You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by fe...@apache.org on 2017/05/04 04:52:30 UTC
spark git commit: [SPARK-20544][SPARKR] skip tests when running on
CRAN
Repository: spark
Updated Branches:
refs/heads/branch-2.2 d8bd213f1 -> 5fe9313d7
[SPARK-20544][SPARKR] skip tests when running on CRAN
General rule on skip or not:
skip if
- RDD tests
- tests could run long or complicated (streaming, hivecontext)
- tests on error conditions
- tests won't likely change/break
unit tests, `R CMD check --as-cran`, `R CMD check`
Author: Felix Cheung <fe...@hotmail.com>
Closes #17817 from felixcheung/rskiptest.
(cherry picked from commit fc472bddd1d9c6a28e57e31496c0166777af597e)
Signed-off-by: Felix Cheung <fe...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5fe9313d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5fe9313d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5fe9313d
Branch: refs/heads/branch-2.2
Commit: 5fe9313d7c81679981000b8aea5ea4668a0a0bc8
Parents: d8bd213
Author: Felix Cheung <fe...@hotmail.com>
Authored: Wed May 3 21:40:18 2017 -0700
Committer: Felix Cheung <fe...@apache.org>
Committed: Wed May 3 21:51:33 2017 -0700
----------------------------------------------------------------------
R/pkg/inst/tests/testthat/test_Serde.R | 6 ++
R/pkg/inst/tests/testthat/test_Windows.R | 2 +
R/pkg/inst/tests/testthat/test_binaryFile.R | 8 ++
.../inst/tests/testthat/test_binary_function.R | 6 ++
R/pkg/inst/tests/testthat/test_broadcast.R | 4 +
R/pkg/inst/tests/testthat/test_client.R | 8 ++
R/pkg/inst/tests/testthat/test_context.R | 16 +++
R/pkg/inst/tests/testthat/test_includePackage.R | 4 +
.../inst/tests/testthat/test_mllib_clustering.R | 4 +
.../inst/tests/testthat/test_mllib_regression.R | 12 +++
.../tests/testthat/test_parallelize_collect.R | 8 ++
R/pkg/inst/tests/testthat/test_rdd.R | 106 ++++++++++++++++++-
R/pkg/inst/tests/testthat/test_shuffle.R | 24 +++++
R/pkg/inst/tests/testthat/test_sparkR.R | 2 +
R/pkg/inst/tests/testthat/test_sparkSQL.R | 60 +++++++++++
R/pkg/inst/tests/testthat/test_streaming.R | 12 +++
R/pkg/inst/tests/testthat/test_take.R | 2 +
R/pkg/inst/tests/testthat/test_textFile.R | 18 ++++
R/pkg/inst/tests/testthat/test_utils.R | 5 +
R/run-tests.sh | 2 +-
20 files changed, 306 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_Serde.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_Serde.R b/R/pkg/inst/tests/testthat/test_Serde.R
index b5f6f1b..518fb7b 100644
--- a/R/pkg/inst/tests/testthat/test_Serde.R
+++ b/R/pkg/inst/tests/testthat/test_Serde.R
@@ -20,6 +20,8 @@ context("SerDe functionality")
sparkSession <- sparkR.session(enableHiveSupport = FALSE)
test_that("SerDe of primitive types", {
+ skip_on_cran()
+
x <- callJStatic("SparkRHandler", "echo", 1L)
expect_equal(x, 1L)
expect_equal(class(x), "integer")
@@ -38,6 +40,8 @@ test_that("SerDe of primitive types", {
})
test_that("SerDe of list of primitive types", {
+ skip_on_cran()
+
x <- list(1L, 2L, 3L)
y <- callJStatic("SparkRHandler", "echo", x)
expect_equal(x, y)
@@ -65,6 +69,8 @@ test_that("SerDe of list of primitive types", {
})
test_that("SerDe of list of lists", {
+ skip_on_cran()
+
x <- list(list(1L, 2L, 3L), list(1, 2, 3),
list(TRUE, FALSE), list("a", "b", "c"))
y <- callJStatic("SparkRHandler", "echo", x)
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_Windows.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_Windows.R b/R/pkg/inst/tests/testthat/test_Windows.R
index 1d777dd..919b063 100644
--- a/R/pkg/inst/tests/testthat/test_Windows.R
+++ b/R/pkg/inst/tests/testthat/test_Windows.R
@@ -17,6 +17,8 @@
context("Windows-specific tests")
test_that("sparkJars tag in SparkContext", {
+ skip_on_cran()
+
if (.Platform$OS.type != "windows") {
skip("This test is only for Windows, skipped")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_binaryFile.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_binaryFile.R b/R/pkg/inst/tests/testthat/test_binaryFile.R
index b5c279e..63f54e1 100644
--- a/R/pkg/inst/tests/testthat/test_binaryFile.R
+++ b/R/pkg/inst/tests/testthat/test_binaryFile.R
@@ -24,6 +24,8 @@ sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext",
mockFile <- c("Spark is pretty.", "Spark is awesome.")
test_that("saveAsObjectFile()/objectFile() following textFile() works", {
+ skip_on_cran()
+
fileName1 <- tempfile(pattern = "spark-test", fileext = ".tmp")
fileName2 <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName1)
@@ -38,6 +40,8 @@ test_that("saveAsObjectFile()/objectFile() following textFile() works", {
})
test_that("saveAsObjectFile()/objectFile() works on a parallelized list", {
+ skip_on_cran()
+
fileName <- tempfile(pattern = "spark-test", fileext = ".tmp")
l <- list(1, 2, 3)
@@ -50,6 +54,8 @@ test_that("saveAsObjectFile()/objectFile() works on a parallelized list", {
})
test_that("saveAsObjectFile()/objectFile() following RDD transformations works", {
+ skip_on_cran()
+
fileName1 <- tempfile(pattern = "spark-test", fileext = ".tmp")
fileName2 <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName1)
@@ -74,6 +80,8 @@ test_that("saveAsObjectFile()/objectFile() following RDD transformations works",
})
test_that("saveAsObjectFile()/objectFile() works with multiple paths", {
+ skip_on_cran()
+
fileName1 <- tempfile(pattern = "spark-test", fileext = ".tmp")
fileName2 <- tempfile(pattern = "spark-test", fileext = ".tmp")
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_binary_function.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_binary_function.R b/R/pkg/inst/tests/testthat/test_binary_function.R
index 59cb2e6..25bb2b8 100644
--- a/R/pkg/inst/tests/testthat/test_binary_function.R
+++ b/R/pkg/inst/tests/testthat/test_binary_function.R
@@ -29,6 +29,8 @@ rdd <- parallelize(sc, nums, 2L)
mockFile <- c("Spark is pretty.", "Spark is awesome.")
test_that("union on two RDDs", {
+ skip_on_cran()
+
actual <- collectRDD(unionRDD(rdd, rdd))
expect_equal(actual, as.list(rep(nums, 2)))
@@ -51,6 +53,8 @@ test_that("union on two RDDs", {
})
test_that("cogroup on two RDDs", {
+ skip_on_cran()
+
rdd1 <- parallelize(sc, list(list(1, 1), list(2, 4)))
rdd2 <- parallelize(sc, list(list(1, 2), list(1, 3)))
cogroup.rdd <- cogroup(rdd1, rdd2, numPartitions = 2L)
@@ -69,6 +73,8 @@ test_that("cogroup on two RDDs", {
})
test_that("zipPartitions() on RDDs", {
+ skip_on_cran()
+
rdd1 <- parallelize(sc, 1:2, 2L) # 1, 2
rdd2 <- parallelize(sc, 1:4, 2L) # 1:2, 3:4
rdd3 <- parallelize(sc, 1:6, 2L) # 1:3, 4:6
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_broadcast.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_broadcast.R b/R/pkg/inst/tests/testthat/test_broadcast.R
index 65f204d..504ded4 100644
--- a/R/pkg/inst/tests/testthat/test_broadcast.R
+++ b/R/pkg/inst/tests/testthat/test_broadcast.R
@@ -26,6 +26,8 @@ nums <- 1:2
rrdd <- parallelize(sc, nums, 2L)
test_that("using broadcast variable", {
+ skip_on_cran()
+
randomMat <- matrix(nrow = 10, ncol = 10, data = rnorm(100))
randomMatBr <- broadcast(sc, randomMat)
@@ -38,6 +40,8 @@ test_that("using broadcast variable", {
})
test_that("without using broadcast variable", {
+ skip_on_cran()
+
randomMat <- matrix(nrow = 10, ncol = 10, data = rnorm(100))
useBroadcast <- function(x) {
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_client.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_client.R b/R/pkg/inst/tests/testthat/test_client.R
index 0cf25fe..3d53beb 100644
--- a/R/pkg/inst/tests/testthat/test_client.R
+++ b/R/pkg/inst/tests/testthat/test_client.R
@@ -18,6 +18,8 @@
context("functions in client.R")
test_that("adding spark-testing-base as a package works", {
+ skip_on_cran()
+
args <- generateSparkSubmitArgs("", "", "", "",
"holdenk:spark-testing-base:1.3.0_0.0.5")
expect_equal(gsub("[[:space:]]", "", args),
@@ -26,16 +28,22 @@ test_that("adding spark-testing-base as a package works", {
})
test_that("no package specified doesn't add packages flag", {
+ skip_on_cran()
+
args <- generateSparkSubmitArgs("", "", "", "", "")
expect_equal(gsub("[[:space:]]", "", args),
"")
})
test_that("multiple packages don't produce a warning", {
+ skip_on_cran()
+
expect_warning(generateSparkSubmitArgs("", "", "", "", c("A", "B")), NA)
})
test_that("sparkJars sparkPackages as character vectors", {
+ skip_on_cran()
+
args <- generateSparkSubmitArgs("", "", c("one.jar", "two.jar", "three.jar"), "",
c("com.databricks:spark-avro_2.10:2.0.1"))
expect_match(args, "--jars one.jar,two.jar,three.jar")
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_context.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R
index c847113..9ec79ad 100644
--- a/R/pkg/inst/tests/testthat/test_context.R
+++ b/R/pkg/inst/tests/testthat/test_context.R
@@ -18,6 +18,8 @@
context("test functions in sparkR.R")
test_that("Check masked functions", {
+ skip_on_cran()
+
# Check that we are not masking any new function from base, stats, testthat unexpectedly
# NOTE: We should avoid adding entries to *namesOfMaskedCompletely* as masked functions make it
# hard for users to use base R functions. Please check when in doubt.
@@ -55,6 +57,8 @@ test_that("Check masked functions", {
})
test_that("repeatedly starting and stopping SparkR", {
+ skip_on_cran()
+
for (i in 1:4) {
sc <- suppressWarnings(sparkR.init())
rdd <- parallelize(sc, 1:20, 2L)
@@ -73,6 +77,8 @@ test_that("repeatedly starting and stopping SparkSession", {
})
test_that("rdd GC across sparkR.stop", {
+ skip_on_cran()
+
sc <- sparkR.sparkContext() # sc should get id 0
rdd1 <- parallelize(sc, 1:20, 2L) # rdd1 should get id 1
rdd2 <- parallelize(sc, 1:10, 2L) # rdd2 should get id 2
@@ -96,6 +102,8 @@ test_that("rdd GC across sparkR.stop", {
})
test_that("job group functions can be called", {
+ skip_on_cran()
+
sc <- sparkR.sparkContext()
setJobGroup("groupId", "job description", TRUE)
cancelJobGroup("groupId")
@@ -108,12 +116,16 @@ test_that("job group functions can be called", {
})
test_that("utility function can be called", {
+ skip_on_cran()
+
sparkR.sparkContext()
setLogLevel("ERROR")
sparkR.session.stop()
})
test_that("getClientModeSparkSubmitOpts() returns spark-submit args from whitelist", {
+ skip_on_cran()
+
e <- new.env()
e[["spark.driver.memory"]] <- "512m"
ops <- getClientModeSparkSubmitOpts("sparkrmain", e)
@@ -141,6 +153,8 @@ test_that("getClientModeSparkSubmitOpts() returns spark-submit args from whiteli
})
test_that("sparkJars sparkPackages as comma-separated strings", {
+ skip_on_cran()
+
expect_warning(processSparkJars(" a, b "))
jars <- suppressWarnings(processSparkJars(" a, b "))
expect_equal(lapply(jars, basename), list("a", "b"))
@@ -168,6 +182,8 @@ test_that("spark.lapply should perform simple transforms", {
})
test_that("add and get file to be downloaded with Spark job on every node", {
+ skip_on_cran()
+
sparkR.sparkContext()
# Test add file.
path <- tempfile(pattern = "hello", fileext = ".txt")
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_includePackage.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_includePackage.R b/R/pkg/inst/tests/testthat/test_includePackage.R
index 563ea29..f823ad8 100644
--- a/R/pkg/inst/tests/testthat/test_includePackage.R
+++ b/R/pkg/inst/tests/testthat/test_includePackage.R
@@ -26,6 +26,8 @@ nums <- 1:2
rdd <- parallelize(sc, nums, 2L)
test_that("include inside function", {
+ skip_on_cran()
+
# Only run the test if plyr is installed.
if ("plyr" %in% rownames(installed.packages())) {
suppressPackageStartupMessages(library(plyr))
@@ -42,6 +44,8 @@ test_that("include inside function", {
})
test_that("use include package", {
+ skip_on_cran()
+
# Only run the test if plyr is installed.
if ("plyr" %in% rownames(installed.packages())) {
suppressPackageStartupMessages(library(plyr))
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_mllib_clustering.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_mllib_clustering.R b/R/pkg/inst/tests/testthat/test_mllib_clustering.R
index 1661e98..478012e 100644
--- a/R/pkg/inst/tests/testthat/test_mllib_clustering.R
+++ b/R/pkg/inst/tests/testthat/test_mllib_clustering.R
@@ -255,6 +255,8 @@ test_that("spark.lda with libsvm", {
})
test_that("spark.lda with text input", {
+ skip_on_cran()
+
text <- read.text(absoluteSparkPath("data/mllib/sample_lda_data.txt"))
model <- spark.lda(text, optimizer = "online", features = "value")
@@ -297,6 +299,8 @@ test_that("spark.lda with text input", {
})
test_that("spark.posterior and spark.perplexity", {
+ skip_on_cran()
+
text <- read.text(absoluteSparkPath("data/mllib/sample_lda_data.txt"))
model <- spark.lda(text, features = "value", k = 3)
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_mllib_regression.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_mllib_regression.R b/R/pkg/inst/tests/testthat/test_mllib_regression.R
index 3e9ad77..58924f9 100644
--- a/R/pkg/inst/tests/testthat/test_mllib_regression.R
+++ b/R/pkg/inst/tests/testthat/test_mllib_regression.R
@@ -23,6 +23,8 @@ context("MLlib regression algorithms, except for tree-based algorithms")
sparkSession <- sparkR.session(enableHiveSupport = FALSE)
test_that("formula of spark.glm", {
+ skip_on_cran()
+
training <- suppressWarnings(createDataFrame(iris))
# directly calling the spark API
# dot minus and intercept vs native glm
@@ -195,6 +197,8 @@ test_that("spark.glm summary", {
})
test_that("spark.glm save/load", {
+ skip_on_cran()
+
training <- suppressWarnings(createDataFrame(iris))
m <- spark.glm(training, Sepal_Width ~ Sepal_Length + Species)
s <- summary(m)
@@ -222,6 +226,8 @@ test_that("spark.glm save/load", {
})
test_that("formula of glm", {
+ skip_on_cran()
+
training <- suppressWarnings(createDataFrame(iris))
# dot minus and intercept vs native glm
model <- glm(Sepal_Width ~ . - Species + 0, data = training)
@@ -248,6 +254,8 @@ test_that("formula of glm", {
})
test_that("glm and predict", {
+ skip_on_cran()
+
training <- suppressWarnings(createDataFrame(iris))
# gaussian family
model <- glm(Sepal_Width ~ Sepal_Length + Species, data = training)
@@ -292,6 +300,8 @@ test_that("glm and predict", {
})
test_that("glm summary", {
+ skip_on_cran()
+
# gaussian family
training <- suppressWarnings(createDataFrame(iris))
stats <- summary(glm(Sepal_Width ~ Sepal_Length + Species, data = training))
@@ -341,6 +351,8 @@ test_that("glm summary", {
})
test_that("glm save/load", {
+ skip_on_cran()
+
training <- suppressWarnings(createDataFrame(iris))
m <- glm(Sepal_Width ~ Sepal_Length + Species, data = training)
s <- summary(m)
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_parallelize_collect.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_parallelize_collect.R b/R/pkg/inst/tests/testthat/test_parallelize_collect.R
index 55972e1..1f7f387 100644
--- a/R/pkg/inst/tests/testthat/test_parallelize_collect.R
+++ b/R/pkg/inst/tests/testthat/test_parallelize_collect.R
@@ -39,6 +39,8 @@ jsc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext",
# Tests
test_that("parallelize() on simple vectors and lists returns an RDD", {
+ skip_on_cran()
+
numVectorRDD <- parallelize(jsc, numVector, 1)
numVectorRDD2 <- parallelize(jsc, numVector, 10)
numListRDD <- parallelize(jsc, numList, 1)
@@ -66,6 +68,8 @@ test_that("parallelize() on simple vectors and lists returns an RDD", {
})
test_that("collect(), following a parallelize(), gives back the original collections", {
+ skip_on_cran()
+
numVectorRDD <- parallelize(jsc, numVector, 10)
expect_equal(collectRDD(numVectorRDD), as.list(numVector))
@@ -86,6 +90,8 @@ test_that("collect(), following a parallelize(), gives back the original collect
})
test_that("regression: collect() following a parallelize() does not drop elements", {
+ skip_on_cran()
+
# 10 %/% 6 = 1, ceiling(10 / 6) = 2
collLen <- 10
numPart <- 6
@@ -95,6 +101,8 @@ test_that("regression: collect() following a parallelize() does not drop element
})
test_that("parallelize() and collect() work for lists of pairs (pairwise data)", {
+ skip_on_cran()
+
# use the pairwise logical to indicate pairwise data
numPairsRDDD1 <- parallelize(jsc, numPairs, 1)
numPairsRDDD2 <- parallelize(jsc, numPairs, 2)
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_rdd.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_rdd.R b/R/pkg/inst/tests/testthat/test_rdd.R
index b72c801..a3b1631 100644
--- a/R/pkg/inst/tests/testthat/test_rdd.R
+++ b/R/pkg/inst/tests/testthat/test_rdd.R
@@ -29,22 +29,30 @@ intPairs <- list(list(1L, -1), list(2L, 100), list(2L, 1), list(1L, 200))
intRdd <- parallelize(sc, intPairs, 2L)
test_that("get number of partitions in RDD", {
+ skip_on_cran()
+
expect_equal(getNumPartitionsRDD(rdd), 2)
expect_equal(getNumPartitionsRDD(intRdd), 2)
})
test_that("first on RDD", {
+ skip_on_cran()
+
expect_equal(firstRDD(rdd), 1)
newrdd <- lapply(rdd, function(x) x + 1)
expect_equal(firstRDD(newrdd), 2)
})
test_that("count and length on RDD", {
- expect_equal(countRDD(rdd), 10)
- expect_equal(lengthRDD(rdd), 10)
+ skip_on_cran()
+
+ expect_equal(countRDD(rdd), 10)
+ expect_equal(lengthRDD(rdd), 10)
})
test_that("count by values and keys", {
+ skip_on_cran()
+
mods <- lapply(rdd, function(x) { x %% 3 })
actual <- countByValue(mods)
expected <- list(list(0, 3L), list(1, 4L), list(2, 3L))
@@ -56,30 +64,40 @@ test_that("count by values and keys", {
})
test_that("lapply on RDD", {
+ skip_on_cran()
+
multiples <- lapply(rdd, function(x) { 2 * x })
actual <- collectRDD(multiples)
expect_equal(actual, as.list(nums * 2))
})
test_that("lapplyPartition on RDD", {
+ skip_on_cran()
+
sums <- lapplyPartition(rdd, function(part) { sum(unlist(part)) })
actual <- collectRDD(sums)
expect_equal(actual, list(15, 40))
})
test_that("mapPartitions on RDD", {
+ skip_on_cran()
+
sums <- mapPartitions(rdd, function(part) { sum(unlist(part)) })
actual <- collectRDD(sums)
expect_equal(actual, list(15, 40))
})
test_that("flatMap() on RDDs", {
+ skip_on_cran()
+
flat <- flatMap(intRdd, function(x) { list(x, x) })
actual <- collectRDD(flat)
expect_equal(actual, rep(intPairs, each = 2))
})
test_that("filterRDD on RDD", {
+ skip_on_cran()
+
filtered.rdd <- filterRDD(rdd, function(x) { x %% 2 == 0 })
actual <- collectRDD(filtered.rdd)
expect_equal(actual, list(2, 4, 6, 8, 10))
@@ -95,6 +113,8 @@ test_that("filterRDD on RDD", {
})
test_that("lookup on RDD", {
+ skip_on_cran()
+
vals <- lookup(intRdd, 1L)
expect_equal(vals, list(-1, 200))
@@ -103,6 +123,8 @@ test_that("lookup on RDD", {
})
test_that("several transformations on RDD (a benchmark on PipelinedRDD)", {
+ skip_on_cran()
+
rdd2 <- rdd
for (i in 1:12)
rdd2 <- lapplyPartitionsWithIndex(
@@ -117,6 +139,8 @@ test_that("several transformations on RDD (a benchmark on PipelinedRDD)", {
})
test_that("PipelinedRDD support actions: cache(), persist(), unpersist(), checkpoint()", {
+ skip_on_cran()
+
# RDD
rdd2 <- rdd
# PipelinedRDD
@@ -158,6 +182,8 @@ test_that("PipelinedRDD support actions: cache(), persist(), unpersist(), checkp
})
test_that("reduce on RDD", {
+ skip_on_cran()
+
sum <- reduce(rdd, "+")
expect_equal(sum, 55)
@@ -167,6 +193,8 @@ test_that("reduce on RDD", {
})
test_that("lapply with dependency", {
+ skip_on_cran()
+
fa <- 5
multiples <- lapply(rdd, function(x) { fa * x })
actual <- collectRDD(multiples)
@@ -175,6 +203,8 @@ test_that("lapply with dependency", {
})
test_that("lapplyPartitionsWithIndex on RDDs", {
+ skip_on_cran()
+
func <- function(partIndex, part) { list(partIndex, Reduce("+", part)) }
actual <- collectRDD(lapplyPartitionsWithIndex(rdd, func), flatten = FALSE)
expect_equal(actual, list(list(0, 15), list(1, 40)))
@@ -191,10 +221,14 @@ test_that("lapplyPartitionsWithIndex on RDDs", {
})
test_that("sampleRDD() on RDDs", {
+ skip_on_cran()
+
expect_equal(unlist(collectRDD(sampleRDD(rdd, FALSE, 1.0, 2014L))), nums)
})
test_that("takeSample() on RDDs", {
+ skip_on_cran()
+
# ported from RDDSuite.scala, modified seeds
data <- parallelize(sc, 1:100, 2L)
for (seed in 4:5) {
@@ -237,6 +271,8 @@ test_that("takeSample() on RDDs", {
})
test_that("mapValues() on pairwise RDDs", {
+ skip_on_cran()
+
multiples <- mapValues(intRdd, function(x) { x * 2 })
actual <- collectRDD(multiples)
expected <- lapply(intPairs, function(x) {
@@ -246,6 +282,8 @@ test_that("mapValues() on pairwise RDDs", {
})
test_that("flatMapValues() on pairwise RDDs", {
+ skip_on_cran()
+
l <- parallelize(sc, list(list(1, c(1, 2)), list(2, c(3, 4))))
actual <- collectRDD(flatMapValues(l, function(x) { x }))
expect_equal(actual, list(list(1, 1), list(1, 2), list(2, 3), list(2, 4)))
@@ -258,6 +296,8 @@ test_that("flatMapValues() on pairwise RDDs", {
})
test_that("reduceByKeyLocally() on PairwiseRDDs", {
+ skip_on_cran()
+
pairs <- parallelize(sc, list(list(1, 2), list(1.1, 3), list(1, 4)), 2L)
actual <- reduceByKeyLocally(pairs, "+")
expect_equal(sortKeyValueList(actual),
@@ -271,6 +311,8 @@ test_that("reduceByKeyLocally() on PairwiseRDDs", {
})
test_that("distinct() on RDDs", {
+ skip_on_cran()
+
nums.rep2 <- rep(1:10, 2)
rdd.rep2 <- parallelize(sc, nums.rep2, 2L)
uniques <- distinctRDD(rdd.rep2)
@@ -279,21 +321,29 @@ test_that("distinct() on RDDs", {
})
test_that("maximum() on RDDs", {
+ skip_on_cran()
+
max <- maximum(rdd)
expect_equal(max, 10)
})
test_that("minimum() on RDDs", {
+ skip_on_cran()
+
min <- minimum(rdd)
expect_equal(min, 1)
})
test_that("sumRDD() on RDDs", {
+ skip_on_cran()
+
sum <- sumRDD(rdd)
expect_equal(sum, 55)
})
test_that("keyBy on RDDs", {
+ skip_on_cran()
+
func <- function(x) { x * x }
keys <- keyBy(rdd, func)
actual <- collectRDD(keys)
@@ -301,6 +351,8 @@ test_that("keyBy on RDDs", {
})
test_that("repartition/coalesce on RDDs", {
+ skip_on_cran()
+
rdd <- parallelize(sc, 1:20, 4L) # each partition contains 5 elements
# repartition
@@ -322,6 +374,8 @@ test_that("repartition/coalesce on RDDs", {
})
test_that("sortBy() on RDDs", {
+ skip_on_cran()
+
sortedRdd <- sortBy(rdd, function(x) { x * x }, ascending = FALSE)
actual <- collectRDD(sortedRdd)
expect_equal(actual, as.list(sort(nums, decreasing = TRUE)))
@@ -333,6 +387,8 @@ test_that("sortBy() on RDDs", {
})
test_that("takeOrdered() on RDDs", {
+ skip_on_cran()
+
l <- list(10, 1, 2, 9, 3, 4, 5, 6, 7)
rdd <- parallelize(sc, l)
actual <- takeOrdered(rdd, 6L)
@@ -345,6 +401,8 @@ test_that("takeOrdered() on RDDs", {
})
test_that("top() on RDDs", {
+ skip_on_cran()
+
l <- list(10, 1, 2, 9, 3, 4, 5, 6, 7)
rdd <- parallelize(sc, l)
actual <- top(rdd, 6L)
@@ -357,6 +415,8 @@ test_that("top() on RDDs", {
})
test_that("fold() on RDDs", {
+ skip_on_cran()
+
actual <- fold(rdd, 0, "+")
expect_equal(actual, Reduce("+", nums, 0))
@@ -366,6 +426,8 @@ test_that("fold() on RDDs", {
})
test_that("aggregateRDD() on RDDs", {
+ skip_on_cran()
+
rdd <- parallelize(sc, list(1, 2, 3, 4))
zeroValue <- list(0, 0)
seqOp <- function(x, y) { list(x[[1]] + y, x[[2]] + 1) }
@@ -379,6 +441,8 @@ test_that("aggregateRDD() on RDDs", {
})
test_that("zipWithUniqueId() on RDDs", {
+ skip_on_cran()
+
rdd <- parallelize(sc, list("a", "b", "c", "d", "e"), 3L)
actual <- collectRDD(zipWithUniqueId(rdd))
expected <- list(list("a", 0), list("b", 1), list("c", 4),
@@ -393,6 +457,8 @@ test_that("zipWithUniqueId() on RDDs", {
})
test_that("zipWithIndex() on RDDs", {
+ skip_on_cran()
+
rdd <- parallelize(sc, list("a", "b", "c", "d", "e"), 3L)
actual <- collectRDD(zipWithIndex(rdd))
expected <- list(list("a", 0), list("b", 1), list("c", 2),
@@ -407,24 +473,32 @@ test_that("zipWithIndex() on RDDs", {
})
test_that("glom() on RDD", {
+ skip_on_cran()
+
rdd <- parallelize(sc, as.list(1:4), 2L)
actual <- collectRDD(glom(rdd))
expect_equal(actual, list(list(1, 2), list(3, 4)))
})
test_that("keys() on RDDs", {
+ skip_on_cran()
+
keys <- keys(intRdd)
actual <- collectRDD(keys)
expect_equal(actual, lapply(intPairs, function(x) { x[[1]] }))
})
test_that("values() on RDDs", {
+ skip_on_cran()
+
values <- values(intRdd)
actual <- collectRDD(values)
expect_equal(actual, lapply(intPairs, function(x) { x[[2]] }))
})
test_that("pipeRDD() on RDDs", {
+ skip_on_cran()
+
actual <- collectRDD(pipeRDD(rdd, "more"))
expected <- as.list(as.character(1:10))
expect_equal(actual, expected)
@@ -442,6 +516,8 @@ test_that("pipeRDD() on RDDs", {
})
test_that("zipRDD() on RDDs", {
+ skip_on_cran()
+
rdd1 <- parallelize(sc, 0:4, 2)
rdd2 <- parallelize(sc, 1000:1004, 2)
actual <- collectRDD(zipRDD(rdd1, rdd2))
@@ -471,6 +547,8 @@ test_that("zipRDD() on RDDs", {
})
test_that("cartesian() on RDDs", {
+ skip_on_cran()
+
rdd <- parallelize(sc, 1:3)
actual <- collectRDD(cartesian(rdd, rdd))
expect_equal(sortKeyValueList(actual),
@@ -514,6 +592,8 @@ test_that("cartesian() on RDDs", {
})
test_that("subtract() on RDDs", {
+ skip_on_cran()
+
l <- list(1, 1, 2, 2, 3, 4)
rdd1 <- parallelize(sc, l)
@@ -541,6 +621,8 @@ test_that("subtract() on RDDs", {
})
test_that("subtractByKey() on pairwise RDDs", {
+ skip_on_cran()
+
l <- list(list("a", 1), list("b", 4),
list("b", 5), list("a", 2))
rdd1 <- parallelize(sc, l)
@@ -570,6 +652,8 @@ test_that("subtractByKey() on pairwise RDDs", {
})
test_that("intersection() on RDDs", {
+ skip_on_cran()
+
# intersection with self
actual <- collectRDD(intersection(rdd, rdd))
expect_equal(sort(as.integer(actual)), nums)
@@ -586,6 +670,8 @@ test_that("intersection() on RDDs", {
})
test_that("join() on pairwise RDDs", {
+ skip_on_cran()
+
rdd1 <- parallelize(sc, list(list(1, 1), list(2, 4)))
rdd2 <- parallelize(sc, list(list(1, 2), list(1, 3)))
actual <- collectRDD(joinRDD(rdd1, rdd2, 2L))
@@ -610,6 +696,8 @@ test_that("join() on pairwise RDDs", {
})
test_that("leftOuterJoin() on pairwise RDDs", {
+ skip_on_cran()
+
rdd1 <- parallelize(sc, list(list(1, 1), list(2, 4)))
rdd2 <- parallelize(sc, list(list(1, 2), list(1, 3)))
actual <- collectRDD(leftOuterJoin(rdd1, rdd2, 2L))
@@ -640,6 +728,8 @@ test_that("leftOuterJoin() on pairwise RDDs", {
})
test_that("rightOuterJoin() on pairwise RDDs", {
+ skip_on_cran()
+
rdd1 <- parallelize(sc, list(list(1, 2), list(1, 3)))
rdd2 <- parallelize(sc, list(list(1, 1), list(2, 4)))
actual <- collectRDD(rightOuterJoin(rdd1, rdd2, 2L))
@@ -667,6 +757,8 @@ test_that("rightOuterJoin() on pairwise RDDs", {
})
test_that("fullOuterJoin() on pairwise RDDs", {
+ skip_on_cran()
+
rdd1 <- parallelize(sc, list(list(1, 2), list(1, 3), list(3, 3)))
rdd2 <- parallelize(sc, list(list(1, 1), list(2, 4)))
actual <- collectRDD(fullOuterJoin(rdd1, rdd2, 2L))
@@ -698,6 +790,8 @@ test_that("fullOuterJoin() on pairwise RDDs", {
})
test_that("sortByKey() on pairwise RDDs", {
+ skip_on_cran()
+
numPairsRdd <- map(rdd, function(x) { list (x, x) })
sortedRdd <- sortByKey(numPairsRdd, ascending = FALSE)
actual <- collectRDD(sortedRdd)
@@ -747,6 +841,8 @@ test_that("sortByKey() on pairwise RDDs", {
})
test_that("collectAsMap() on a pairwise RDD", {
+ skip_on_cran()
+
rdd <- parallelize(sc, list(list(1, 2), list(3, 4)))
vals <- collectAsMap(rdd)
expect_equal(vals, list(`1` = 2, `3` = 4))
@@ -765,11 +861,15 @@ test_that("collectAsMap() on a pairwise RDD", {
})
test_that("show()", {
+ skip_on_cran()
+
rdd <- parallelize(sc, list(1:10))
expect_output(showRDD(rdd), "ParallelCollectionRDD\\[\\d+\\] at parallelize at RRDD\\.scala:\\d+")
})
test_that("sampleByKey() on pairwise RDDs", {
+ skip_on_cran()
+
rdd <- parallelize(sc, 1:2000)
pairsRDD <- lapply(rdd, function(x) { if (x %% 2 == 0) list("a", x) else list("b", x) })
fractions <- list(a = 0.2, b = 0.1)
@@ -794,6 +894,8 @@ test_that("sampleByKey() on pairwise RDDs", {
})
test_that("Test correct concurrency of RRDD.compute()", {
+ skip_on_cran()
+
rdd <- parallelize(sc, 1:1000, 100)
jrdd <- getJRDD(lapply(rdd, function(x) { x }), "row")
zrdd <- callJMethod(jrdd, "zip", jrdd)
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_shuffle.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_shuffle.R b/R/pkg/inst/tests/testthat/test_shuffle.R
index d38efab..cedf4f1 100644
--- a/R/pkg/inst/tests/testthat/test_shuffle.R
+++ b/R/pkg/inst/tests/testthat/test_shuffle.R
@@ -37,6 +37,8 @@ strList <- list("Dexter Morgan: Blood. Sometimes it sets my teeth on edge and ",
strListRDD <- parallelize(sc, strList, 4)
test_that("groupByKey for integers", {
+ skip_on_cran()
+
grouped <- groupByKey(intRdd, 2L)
actual <- collectRDD(grouped)
@@ -46,6 +48,8 @@ test_that("groupByKey for integers", {
})
test_that("groupByKey for doubles", {
+ skip_on_cran()
+
grouped <- groupByKey(doubleRdd, 2L)
actual <- collectRDD(grouped)
@@ -55,6 +59,8 @@ test_that("groupByKey for doubles", {
})
test_that("reduceByKey for ints", {
+ skip_on_cran()
+
reduced <- reduceByKey(intRdd, "+", 2L)
actual <- collectRDD(reduced)
@@ -64,6 +70,8 @@ test_that("reduceByKey for ints", {
})
test_that("reduceByKey for doubles", {
+ skip_on_cran()
+
reduced <- reduceByKey(doubleRdd, "+", 2L)
actual <- collectRDD(reduced)
@@ -72,6 +80,8 @@ test_that("reduceByKey for doubles", {
})
test_that("combineByKey for ints", {
+ skip_on_cran()
+
reduced <- combineByKey(intRdd, function(x) { x }, "+", "+", 2L)
actual <- collectRDD(reduced)
@@ -81,6 +91,8 @@ test_that("combineByKey for ints", {
})
test_that("combineByKey for doubles", {
+ skip_on_cran()
+
reduced <- combineByKey(doubleRdd, function(x) { x }, "+", "+", 2L)
actual <- collectRDD(reduced)
@@ -89,6 +101,8 @@ test_that("combineByKey for doubles", {
})
test_that("combineByKey for characters", {
+ skip_on_cran()
+
stringKeyRDD <- parallelize(sc,
list(list("max", 1L), list("min", 2L),
list("other", 3L), list("max", 4L)), 2L)
@@ -101,6 +115,8 @@ test_that("combineByKey for characters", {
})
test_that("aggregateByKey", {
+ skip_on_cran()
+
# test aggregateByKey for int keys
rdd <- parallelize(sc, list(list(1, 1), list(1, 2), list(2, 3), list(2, 4)))
@@ -129,6 +145,8 @@ test_that("aggregateByKey", {
})
test_that("foldByKey", {
+ skip_on_cran()
+
# test foldByKey for int keys
folded <- foldByKey(intRdd, 0, "+", 2L)
@@ -172,6 +190,8 @@ test_that("foldByKey", {
})
test_that("partitionBy() partitions data correctly", {
+ skip_on_cran()
+
# Partition by magnitude
partitionByMagnitude <- function(key) { if (key >= 3) 1 else 0 }
@@ -187,6 +207,8 @@ test_that("partitionBy() partitions data correctly", {
})
test_that("partitionBy works with dependencies", {
+ skip_on_cran()
+
kOne <- 1
partitionByParity <- function(key) { if (key %% 2 == kOne) 7 else 4 }
@@ -205,6 +227,8 @@ test_that("partitionBy works with dependencies", {
})
test_that("test partitionBy with string keys", {
+ skip_on_cran()
+
words <- flatMap(strListRDD, function(line) { strsplit(line, " ")[[1]] })
wordCount <- lapply(words, function(word) { list(word, 1L) })
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_sparkR.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_sparkR.R b/R/pkg/inst/tests/testthat/test_sparkR.R
index f73fc6b..a40981c 100644
--- a/R/pkg/inst/tests/testthat/test_sparkR.R
+++ b/R/pkg/inst/tests/testthat/test_sparkR.R
@@ -18,6 +18,8 @@
context("functions in sparkR.R")
test_that("sparkCheckInstall", {
+ skip_on_cran()
+
# "local, yarn-client, mesos-client" mode, SPARK_HOME was set correctly,
# and the SparkR job was submitted by "spark-submit"
sparkHome <- paste0(tempdir(), "/", "sparkHome")
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_sparkSQL.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R
index 6a6c9a8..18ff585 100644
--- a/R/pkg/inst/tests/testthat/test_sparkSQL.R
+++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R
@@ -97,15 +97,21 @@ mapTypeJsonPath <- tempfile(pattern = "sparkr-test", fileext = ".tmp")
writeLines(mockLinesMapType, mapTypeJsonPath)
test_that("calling sparkRSQL.init returns existing SQL context", {
+ skip_on_cran()
+
sqlContext <- suppressWarnings(sparkRSQL.init(sc))
expect_equal(suppressWarnings(sparkRSQL.init(sc)), sqlContext)
})
test_that("calling sparkRSQL.init returns existing SparkSession", {
+ skip_on_cran()
+
expect_equal(suppressWarnings(sparkRSQL.init(sc)), sparkSession)
})
test_that("calling sparkR.session returns existing SparkSession", {
+ skip_on_cran()
+
expect_equal(sparkR.session(), sparkSession)
})
@@ -194,6 +200,8 @@ test_that("structField type strings", {
})
test_that("create DataFrame from RDD", {
+ skip_on_cran()
+
rdd <- lapply(parallelize(sc, 1:10), function(x) { list(x, as.character(x)) })
df <- createDataFrame(rdd, list("a", "b"))
dfAsDF <- as.DataFrame(rdd, list("a", "b"))
@@ -291,6 +299,8 @@ test_that("create DataFrame from RDD", {
})
test_that("createDataFrame uses files for large objects", {
+ skip_on_cran()
+
# To simulate a large file scenario, we set spark.r.maxAllocationLimit to a smaller value
conf <- callJMethod(sparkSession, "conf")
callJMethod(conf, "set", "spark.r.maxAllocationLimit", "100")
@@ -351,6 +361,8 @@ test_that("read/write csv as DataFrame", {
})
test_that("Support other types for options", {
+ skip_on_cran()
+
csvPath <- tempfile(pattern = "sparkr-test", fileext = ".csv")
mockLinesCsv <- c("year,make,model,comment,blank",
"\"2012\",\"Tesla\",\"S\",\"No comment\",",
@@ -405,6 +417,8 @@ test_that("convert NAs to null type in DataFrames", {
})
test_that("toDF", {
+ skip_on_cran()
+
rdd <- lapply(parallelize(sc, 1:10), function(x) { list(x, as.character(x)) })
df <- toDF(rdd, list("a", "b"))
expect_is(df, "SparkDataFrame")
@@ -516,6 +530,8 @@ test_that("create DataFrame with complex types", {
})
test_that("create DataFrame from a data.frame with complex types", {
+ skip_on_cran()
+
ldf <- data.frame(row.names = 1:2)
ldf$a_list <- list(list(1, 2), list(3, 4))
ldf$an_envir <- c(as.environment(list(a = 1, b = 2)), as.environment(list(c = 3)))
@@ -528,6 +544,8 @@ test_that("create DataFrame from a data.frame with complex types", {
})
test_that("Collect DataFrame with complex types", {
+ skip_on_cran()
+
# ArrayType
df <- read.json(complexTypeJsonPath)
ldf <- collect(df)
@@ -615,6 +633,8 @@ test_that("read/write json files", {
})
test_that("read/write json files - compression option", {
+ skip_on_cran()
+
df <- read.df(jsonPath, "json")
jsonPath <- tempfile(pattern = "jsonPath", fileext = ".json")
@@ -628,6 +648,8 @@ test_that("read/write json files - compression option", {
})
test_that("jsonRDD() on a RDD with json string", {
+ skip_on_cran()
+
sqlContext <- suppressWarnings(sparkRSQL.init(sc))
rdd <- parallelize(sc, mockLines)
expect_equal(countRDD(rdd), 3)
@@ -684,6 +706,8 @@ test_that(
})
test_that("test cache, uncache and clearCache", {
+ skip_on_cran()
+
df <- read.json(jsonPath)
createOrReplaceTempView(df, "table1")
cacheTable("table1")
@@ -737,6 +761,8 @@ test_that("tableToDF() returns a new DataFrame", {
})
test_that("toRDD() returns an RRDD", {
+ skip_on_cran()
+
df <- read.json(jsonPath)
testRDD <- toRDD(df)
expect_is(testRDD, "RDD")
@@ -744,6 +770,8 @@ test_that("toRDD() returns an RRDD", {
})
test_that("union on two RDDs created from DataFrames returns an RRDD", {
+ skip_on_cran()
+
df <- read.json(jsonPath)
RDD1 <- toRDD(df)
RDD2 <- toRDD(df)
@@ -754,6 +782,8 @@ test_that("union on two RDDs created from DataFrames returns an RRDD", {
})
test_that("union on mixed serialization types correctly returns a byte RRDD", {
+ skip_on_cran()
+
# Byte RDD
nums <- 1:10
rdd <- parallelize(sc, nums, 2L)
@@ -783,6 +813,8 @@ test_that("union on mixed serialization types correctly returns a byte RRDD", {
})
test_that("objectFile() works with row serialization", {
+ skip_on_cran()
+
objectPath <- tempfile(pattern = "spark-test", fileext = ".tmp")
df <- read.json(jsonPath)
dfRDD <- toRDD(df)
@@ -795,6 +827,8 @@ test_that("objectFile() works with row serialization", {
})
test_that("lapply() on a DataFrame returns an RDD with the correct columns", {
+ skip_on_cran()
+
df <- read.json(jsonPath)
testRDD <- lapply(df, function(row) {
row$newCol <- row$age + 5
@@ -863,6 +897,8 @@ test_that("collect() support Unicode characters", {
})
test_that("multiple pipeline transformations result in an RDD with the correct values", {
+ skip_on_cran()
+
df <- read.json(jsonPath)
first <- lapply(df, function(row) {
row$age <- row$age + 5
@@ -2049,6 +2085,8 @@ test_that("mutate(), transform(), rename() and names()", {
})
test_that("read/write ORC files", {
+ skip_on_cran()
+
setHiveContext(sc)
df <- read.df(jsonPath, "json")
@@ -2070,6 +2108,8 @@ test_that("read/write ORC files", {
})
test_that("read/write ORC files - compression option", {
+ skip_on_cran()
+
setHiveContext(sc)
df <- read.df(jsonPath, "json")
@@ -2116,6 +2156,8 @@ test_that("read/write Parquet files", {
})
test_that("read/write Parquet files - compression option/mode", {
+ skip_on_cran()
+
df <- read.df(jsonPath, "json")
tempPath <- tempfile(pattern = "tempPath", fileext = ".parquet")
@@ -2133,6 +2175,8 @@ test_that("read/write Parquet files - compression option/mode", {
})
test_that("read/write text files", {
+ skip_on_cran()
+
# Test write.df and read.df
df <- read.df(jsonPath, "text")
expect_is(df, "SparkDataFrame")
@@ -2154,6 +2198,8 @@ test_that("read/write text files", {
})
test_that("read/write text files - compression option", {
+ skip_on_cran()
+
df <- read.df(jsonPath, "text")
textPath <- tempfile(pattern = "textPath", fileext = ".txt")
@@ -2387,6 +2433,8 @@ test_that("approxQuantile() on a DataFrame", {
})
test_that("SQL error message is returned from JVM", {
+ skip_on_cran()
+
retError <- tryCatch(sql("select * from blah"), error = function(e) e)
expect_equal(grepl("Table or view not found", retError), TRUE)
expect_equal(grepl("blah", retError), TRUE)
@@ -2395,6 +2443,8 @@ test_that("SQL error message is returned from JVM", {
irisDF <- suppressWarnings(createDataFrame(iris))
test_that("Method as.data.frame as a synonym for collect()", {
+ skip_on_cran()
+
expect_equal(as.data.frame(irisDF), collect(irisDF))
irisDF2 <- irisDF[irisDF$Species == "setosa", ]
expect_equal(as.data.frame(irisDF2), collect(irisDF2))
@@ -2812,6 +2862,8 @@ test_that("Window functions on a DataFrame", {
})
test_that("createDataFrame sqlContext parameter backward compatibility", {
+ skip_on_cran()
+
sqlContext <- suppressWarnings(sparkRSQL.init(sc))
a <- 1:3
b <- c("a", "b", "c")
@@ -2891,6 +2943,8 @@ test_that("Setting and getting config on SparkSession, sparkR.conf(), sparkR.uiW
})
test_that("enableHiveSupport on SparkSession", {
+ skip_on_cran()
+
setHiveContext(sc)
unsetHiveContext()
# if we are still here, it must be built with hive
@@ -2906,6 +2960,8 @@ test_that("Spark version from SparkSession", {
})
test_that("Call DataFrameWriter.save() API in Java without path and check argument types", {
+ skip_on_cran()
+
df <- read.df(jsonPath, "json")
# This tests if the exception is thrown from JVM not from SparkR side.
# It makes sure that we can omit path argument in write.df API and then it calls
@@ -2932,6 +2988,8 @@ test_that("Call DataFrameWriter.save() API in Java without path and check argume
})
test_that("Call DataFrameWriter.load() API in Java without path and check argument types", {
+ skip_on_cran()
+
# This tests if the exception is thrown from JVM not from SparkR side.
# It makes sure that we can omit path argument in read.df API and then it calls
# DataFrameWriter.load() without path.
@@ -3056,6 +3114,8 @@ compare_list <- function(list1, list2) {
# This should always be the **very last test** in this test file.
test_that("No extra files are created in SPARK_HOME by starting session and making calls", {
+ skip_on_cran()
+
# Check that it is not creating any extra file.
# Does not check the tempdir which would be cleaned up after.
filesAfter <- list.files(path = sparkRDir, all.files = TRUE)
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_streaming.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_streaming.R b/R/pkg/inst/tests/testthat/test_streaming.R
index b125cb0..8843991 100644
--- a/R/pkg/inst/tests/testthat/test_streaming.R
+++ b/R/pkg/inst/tests/testthat/test_streaming.R
@@ -47,6 +47,8 @@ schema <- structType(structField("name", "string"),
structField("count", "double"))
test_that("read.stream, write.stream, awaitTermination, stopQuery", {
+ skip_on_cran()
+
df <- read.stream("json", path = jsonDir, schema = schema, maxFilesPerTrigger = 1)
expect_true(isStreaming(df))
counts <- count(group_by(df, "name"))
@@ -65,6 +67,8 @@ test_that("read.stream, write.stream, awaitTermination, stopQuery", {
})
test_that("print from explain, lastProgress, status, isActive", {
+ skip_on_cran()
+
df <- read.stream("json", path = jsonDir, schema = schema)
expect_true(isStreaming(df))
counts <- count(group_by(df, "name"))
@@ -83,6 +87,8 @@ test_that("print from explain, lastProgress, status, isActive", {
})
test_that("Stream other format", {
+ skip_on_cran()
+
parquetPath <- tempfile(pattern = "sparkr-test", fileext = ".parquet")
df <- read.df(jsonPath, "json", schema)
write.df(df, parquetPath, "parquet", "overwrite")
@@ -108,6 +114,8 @@ test_that("Stream other format", {
})
test_that("Non-streaming DataFrame", {
+ skip_on_cran()
+
c <- as.DataFrame(cars)
expect_false(isStreaming(c))
@@ -117,6 +125,8 @@ test_that("Non-streaming DataFrame", {
})
test_that("Unsupported operation", {
+ skip_on_cran()
+
# memory sink without aggregation
df <- read.stream("json", path = jsonDir, schema = schema, maxFilesPerTrigger = 1)
expect_error(write.stream(df, "memory", queryName = "people", outputMode = "complete"),
@@ -125,6 +135,8 @@ test_that("Unsupported operation", {
})
test_that("Terminated by error", {
+ skip_on_cran()
+
df <- read.stream("json", path = jsonDir, schema = schema, maxFilesPerTrigger = -1)
counts <- count(group_by(df, "name"))
# This would not fail before returning with a StreamingQuery,
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_take.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_take.R b/R/pkg/inst/tests/testthat/test_take.R
index aaa5328..e2130ea 100644
--- a/R/pkg/inst/tests/testthat/test_take.R
+++ b/R/pkg/inst/tests/testthat/test_take.R
@@ -34,6 +34,8 @@ sparkSession <- sparkR.session(enableHiveSupport = FALSE)
sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", sparkSession)
test_that("take() gives back the original elements in correct count and order", {
+ skip_on_cran()
+
numVectorRDD <- parallelize(sc, numVector, 10)
# case: number of elements to take is less than the size of the first partition
expect_equal(takeRDD(numVectorRDD, 1), as.list(head(numVector, n = 1)))
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_textFile.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_textFile.R b/R/pkg/inst/tests/testthat/test_textFile.R
index 3b46606..28b7e8e 100644
--- a/R/pkg/inst/tests/testthat/test_textFile.R
+++ b/R/pkg/inst/tests/testthat/test_textFile.R
@@ -24,6 +24,8 @@ sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext",
mockFile <- c("Spark is pretty.", "Spark is awesome.")
test_that("textFile() on a local file returns an RDD", {
+ skip_on_cran()
+
fileName <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName)
@@ -36,6 +38,8 @@ test_that("textFile() on a local file returns an RDD", {
})
test_that("textFile() followed by a collect() returns the same content", {
+ skip_on_cran()
+
fileName <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName)
@@ -46,6 +50,8 @@ test_that("textFile() followed by a collect() returns the same content", {
})
test_that("textFile() word count works as expected", {
+ skip_on_cran()
+
fileName <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName)
@@ -64,6 +70,8 @@ test_that("textFile() word count works as expected", {
})
test_that("several transformations on RDD created by textFile()", {
+ skip_on_cran()
+
fileName <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName)
@@ -78,6 +86,8 @@ test_that("several transformations on RDD created by textFile()", {
})
test_that("textFile() followed by a saveAsTextFile() returns the same content", {
+ skip_on_cran()
+
fileName1 <- tempfile(pattern = "spark-test", fileext = ".tmp")
fileName2 <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName1)
@@ -92,6 +102,8 @@ test_that("textFile() followed by a saveAsTextFile() returns the same content",
})
test_that("saveAsTextFile() on a parallelized list works as expected", {
+ skip_on_cran()
+
fileName <- tempfile(pattern = "spark-test", fileext = ".tmp")
l <- list(1, 2, 3)
rdd <- parallelize(sc, l, 1L)
@@ -103,6 +115,8 @@ test_that("saveAsTextFile() on a parallelized list works as expected", {
})
test_that("textFile() and saveAsTextFile() word count works as expected", {
+ skip_on_cran()
+
fileName1 <- tempfile(pattern = "spark-test", fileext = ".tmp")
fileName2 <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName1)
@@ -128,6 +142,8 @@ test_that("textFile() and saveAsTextFile() word count works as expected", {
})
test_that("textFile() on multiple paths", {
+ skip_on_cran()
+
fileName1 <- tempfile(pattern = "spark-test", fileext = ".tmp")
fileName2 <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines("Spark is pretty.", fileName1)
@@ -141,6 +157,8 @@ test_that("textFile() on multiple paths", {
})
test_that("Pipelined operations on RDDs created using textFile", {
+ skip_on_cran()
+
fileName <- tempfile(pattern = "spark-test", fileext = ".tmp")
writeLines(mockFile, fileName)
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/pkg/inst/tests/testthat/test_utils.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_utils.R b/R/pkg/inst/tests/testthat/test_utils.R
index 6d006ec..bda4792 100644
--- a/R/pkg/inst/tests/testthat/test_utils.R
+++ b/R/pkg/inst/tests/testthat/test_utils.R
@@ -23,6 +23,7 @@ sc <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext",
test_that("convertJListToRList() gives back (deserializes) the original JLists
of strings and integers", {
+ skip_on_cran()
# It's hard to manually create a Java List using rJava, since it does not
# support generics well. Instead, we rely on collectRDD() returning a
# JList.
@@ -40,6 +41,7 @@ test_that("convertJListToRList() gives back (deserializes) the original JLists
})
test_that("serializeToBytes on RDD", {
+ skip_on_cran()
# File content
mockFile <- c("Spark is pretty.", "Spark is awesome.")
fileName <- tempfile(pattern = "spark-test", fileext = ".tmp")
@@ -167,6 +169,7 @@ test_that("convertToJSaveMode", {
})
test_that("captureJVMException", {
+ skip_on_cran()
method <- "getSQLDataType"
expect_error(tryCatch(callJStatic("org.apache.spark.sql.api.r.SQLUtils", method,
"unknown"),
@@ -177,6 +180,8 @@ test_that("captureJVMException", {
})
test_that("hashCode", {
+ skip_on_cran()
+
expect_error(hashCode("bc53d3605e8a5b7de1e8e271c2317645"), NA)
})
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe9313d/R/run-tests.sh
----------------------------------------------------------------------
diff --git a/R/run-tests.sh b/R/run-tests.sh
index 742a2c5..29764f4 100755
--- a/R/run-tests.sh
+++ b/R/run-tests.sh
@@ -23,7 +23,7 @@ FAILED=0
LOGFILE=$FWDIR/unit-tests.out
rm -f $LOGFILE
-SPARK_TESTING=1 $FWDIR/../bin/spark-submit --driver-java-options "-Dlog4j.configuration=file:$FWDIR/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
+SPARK_TESTING=1 NOT_CRAN=true $FWDIR/../bin/spark-submit --driver-java-options "-Dlog4j.configuration=file:$FWDIR/log4j.properties" --conf spark.hadoop.fs.defaultFS="file:///" $FWDIR/pkg/tests/run-all.R 2>&1 | tee -a $LOGFILE
FAILED=$((PIPESTATUS[0]||$FAILED))
NUM_TEST_WARNING="$(grep -c -e 'Warnings ----------------' $LOGFILE)"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org