You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sh...@apache.org on 2015/08/28 09:37:55 UTC
spark git commit: [SPARK-10328] [SPARKR] Fix generic for na.omit
Repository: spark
Updated Branches:
refs/heads/master 7583681e6 -> 2f99c3727
[SPARK-10328] [SPARKR] Fix generic for na.omit
S3 function is at https://stat.ethz.ch/R-manual/R-patched/library/stats/html/na.fail.html
Author: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Author: Shivaram Venkataraman <sh...@gmail.com>
Author: Yu ISHIKAWA <yu...@gmail.com>
Closes #8495 from shivaram/na-omit-fix.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2f99c372
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2f99c372
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2f99c372
Branch: refs/heads/master
Commit: 2f99c37273c1d82e2ba39476e4429ea4aaba7ec6
Parents: 7583681
Author: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Authored: Fri Aug 28 00:37:50 2015 -0700
Committer: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Committed: Fri Aug 28 00:37:50 2015 -0700
----------------------------------------------------------------------
R/pkg/R/DataFrame.R | 6 +++---
R/pkg/R/generics.R | 2 +-
R/pkg/inst/tests/test_sparkSQL.R | 23 ++++++++++++++++++++++-
dev/lint-r | 2 +-
4 files changed, 27 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/2f99c372/R/pkg/R/DataFrame.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index dd8126a..74de7c8 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -1699,9 +1699,9 @@ setMethod("dropna",
#' @name na.omit
#' @export
setMethod("na.omit",
- signature(x = "DataFrame"),
- function(x, how = c("any", "all"), minNonNulls = NULL, cols = NULL) {
- dropna(x, how, minNonNulls, cols)
+ signature(object = "DataFrame"),
+ function(object, how = c("any", "all"), minNonNulls = NULL, cols = NULL) {
+ dropna(object, how, minNonNulls, cols)
})
#' fillna
http://git-wip-us.apache.org/repos/asf/spark/blob/2f99c372/R/pkg/R/generics.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R
index a829d46..b578b87 100644
--- a/R/pkg/R/generics.R
+++ b/R/pkg/R/generics.R
@@ -413,7 +413,7 @@ setGeneric("dropna",
#' @rdname nafunctions
#' @export
setGeneric("na.omit",
- function(x, how = c("any", "all"), minNonNulls = NULL, cols = NULL) {
+ function(object, ...) {
standardGeneric("na.omit")
})
http://git-wip-us.apache.org/repos/asf/spark/blob/2f99c372/R/pkg/inst/tests/test_sparkSQL.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/test_sparkSQL.R b/R/pkg/inst/tests/test_sparkSQL.R
index 4b672e1..933b11c 100644
--- a/R/pkg/inst/tests/test_sparkSQL.R
+++ b/R/pkg/inst/tests/test_sparkSQL.R
@@ -1083,7 +1083,7 @@ test_that("describe() and summarize() on a DataFrame", {
expect_equal(collect(stats2)[5, "age"], "30")
})
-test_that("dropna() on a DataFrame", {
+test_that("dropna() and na.omit() on a DataFrame", {
df <- jsonFile(sqlContext, jsonPathNa)
rows <- collect(df)
@@ -1092,6 +1092,8 @@ test_that("dropna() on a DataFrame", {
expected <- rows[!is.na(rows$name),]
actual <- collect(dropna(df, cols = "name"))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df, cols = "name"))
+ expect_identical(expected, actual)
expected <- rows[!is.na(rows$age),]
actual <- collect(dropna(df, cols = "age"))
@@ -1101,48 +1103,67 @@ test_that("dropna() on a DataFrame", {
expect_identical(expected$age, actual$age)
expect_identical(expected$height, actual$height)
expect_identical(expected$name, actual$name)
+ actual <- collect(na.omit(df, cols = "age"))
expected <- rows[!is.na(rows$age) & !is.na(rows$height),]
actual <- collect(dropna(df, cols = c("age", "height")))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df, cols = c("age", "height")))
+ expect_identical(expected, actual)
expected <- rows[!is.na(rows$age) & !is.na(rows$height) & !is.na(rows$name),]
actual <- collect(dropna(df))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df))
+ expect_identical(expected, actual)
# drop with how
expected <- rows[!is.na(rows$age) & !is.na(rows$height) & !is.na(rows$name),]
actual <- collect(dropna(df))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df))
+ expect_identical(expected, actual)
expected <- rows[!is.na(rows$age) | !is.na(rows$height) | !is.na(rows$name),]
actual <- collect(dropna(df, "all"))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df, "all"))
+ expect_identical(expected, actual)
expected <- rows[!is.na(rows$age) & !is.na(rows$height) & !is.na(rows$name),]
actual <- collect(dropna(df, "any"))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df, "any"))
+ expect_identical(expected, actual)
expected <- rows[!is.na(rows$age) & !is.na(rows$height),]
actual <- collect(dropna(df, "any", cols = c("age", "height")))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df, "any", cols = c("age", "height")))
+ expect_identical(expected, actual)
expected <- rows[!is.na(rows$age) | !is.na(rows$height),]
actual <- collect(dropna(df, "all", cols = c("age", "height")))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df, "all", cols = c("age", "height")))
+ expect_identical(expected, actual)
# drop with threshold
expected <- rows[as.integer(!is.na(rows$age)) + as.integer(!is.na(rows$height)) >= 2,]
actual <- collect(dropna(df, minNonNulls = 2, cols = c("age", "height")))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df, minNonNulls = 2, cols = c("age", "height")))
+ expect_identical(expected, actual)
expected <- rows[as.integer(!is.na(rows$age)) +
as.integer(!is.na(rows$height)) +
as.integer(!is.na(rows$name)) >= 3,]
actual <- collect(dropna(df, minNonNulls = 3, cols = c("name", "age", "height")))
expect_identical(expected, actual)
+ actual <- collect(na.omit(df, minNonNulls = 3, cols = c("name", "age", "height")))
+ expect_identical(expected, actual)
})
test_that("fillna() on a DataFrame", {
http://git-wip-us.apache.org/repos/asf/spark/blob/2f99c372/dev/lint-r
----------------------------------------------------------------------
diff --git a/dev/lint-r b/dev/lint-r
index c15d57a..bfda0bc 100755
--- a/dev/lint-r
+++ b/dev/lint-r
@@ -29,7 +29,7 @@ fi
`which Rscript` --vanilla "$SPARK_ROOT_DIR/dev/lint-r.R" "$SPARK_ROOT_DIR" | tee "$LINT_R_REPORT_FILE_NAME"
-NUM_LINES=`wc -l < "$LINT_R_REPORT_FILE_NAME"`
+NUM_LINES=`wc -l < "$LINT_R_REPORT_FILE_NAME" | awk '{print $1}'`
if [ "$NUM_LINES" = "0" ] ; then
lint_status=0
echo "lintr checks passed."
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org