You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2020/04/27 08:04:58 UTC
[spark] branch branch-3.0 updated: [SPARK-31568][R][DOCS] Add
detail about func/key in gapply to documentation
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 7460595 [SPARK-31568][R][DOCS] Add detail about func/key in gapply to documentation
7460595 is described below
commit 74605950da6f115bf491d2371c30f58689b26623
Author: Michael Chirico <mi...@grabtaxi.com>
AuthorDate: Mon Apr 27 17:02:18 2020 +0900
[SPARK-31568][R][DOCS] Add detail about func/key in gapply to documentation
Improve documentation for `gapply` in `SparkR`
Spent a long time this weekend trying to figure out just what exactly `key` is in `gapply`'s `func`. I had assumed it would be a _named_ list, but apparently not -- the examples are working because `schema` is applying the name and the names of the output `data.frame` don't matter.
As near as I can tell the description I've added is correct, namely, that `key` is an unnamed list.
No? Not in code. Only documentation.
Not. Documentation only
Closes #28350 from MichaelChirico/r-gapply-key-doc.
Authored-by: Michael Chirico <mi...@grabtaxi.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
R/pkg/R/DataFrame.R | 112 +++++++++++++++++++++++++++++++---------------------
1 file changed, 67 insertions(+), 45 deletions(-)
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 14d2076..2ebd42a 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -1657,9 +1657,7 @@ setMethod("dapplyCollect",
#'
#' @param cols grouping columns.
#' @param func a function to be applied to each group partition specified by grouping
-#' column of the SparkDataFrame. The function \code{func} takes as argument
-#' a key - grouping columns and a data frame - a local R data.frame.
-#' The output of \code{func} is a local R data.frame.
+#' column of the SparkDataFrame. See Details.
#' @param schema the schema of the resulting SparkDataFrame after the function is applied.
#' The schema must match to output of \code{func}. It has to be defined for each
#' output column with preferred output column name and corresponding data type.
@@ -1669,29 +1667,43 @@ setMethod("dapplyCollect",
#' @aliases gapply,SparkDataFrame-method
#' @rdname gapply
#' @name gapply
+#' @details
+#' \code{func} is a function of two arguments. The first, usually named \code{key}
+#' (though this is not enforced) corresponds to the grouping key, will be an
+#' unnamed \code{list} of \code{length(cols)} length-one objects corresponding
+#' to the grouping columns' values for the current group.
+#'
+#' The second, herein \code{x}, will be a local \code{\link{data.frame}} with the
+#' columns of the input not in \code{cols} for the rows corresponding to \code{key}.
+#'
+#' The output of \code{func} must be a \code{data.frame} matching \code{schema} --
+#' in particular this means the names of the output \code{data.frame} are irrelevant
+#'
#' @seealso \link{gapplyCollect}
#' @examples
#'
#' \dontrun{
-#' Computes the arithmetic mean of the second column by grouping
-#' on the first and third columns. Output the grouping values and the average.
+#' # Computes the arithmetic mean of the second column by grouping
+#' # on the first and third columns. Output the grouping values and the average.
#'
#' df <- createDataFrame (
#' list(list(1L, 1, "1", 0.1), list(1L, 2, "1", 0.2), list(3L, 3, "3", 0.3)),
#' c("a", "b", "c", "d"))
#'
-#' Here our output contains three columns, the key which is a combination of two
-#' columns with data types integer and string and the mean which is a double.
+#' # Here our output contains three columns, the key which is a combination of two
+#' # columns with data types integer and string and the mean which is a double.
#' schema <- structType(structField("a", "integer"), structField("c", "string"),
#' structField("avg", "double"))
#' result <- gapply(
#' df,
#' c("a", "c"),
#' function(key, x) {
+#' # key will either be list(1L, '1') (for the group where a=1L,c='1') or
+#' # list(3L, '3') (for the group where a=3L,c='3')
#' y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE)
#' }, schema)
#'
-#' The schema also can be specified in a DDL-formatted string.
+#' # The schema also can be specified in a DDL-formatted string.
#' schema <- "a INT, c STRING, avg DOUBLE"
#' result <- gapply(
#' df,
@@ -1700,8 +1712,8 @@ setMethod("dapplyCollect",
#' y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE)
#' }, schema)
#'
-#' We can also group the data and afterwards call gapply on GroupedData.
-#' For Example:
+#' # We can also group the data and afterwards call gapply on GroupedData.
+#' # For example:
#' gdf <- group_by(df, "a", "c")
#' result <- gapply(
#' gdf,
@@ -1710,15 +1722,15 @@ setMethod("dapplyCollect",
#' }, schema)
#' collect(result)
#'
-#' Result
-#' ------
-#' a c avg
-#' 3 3 3.0
-#' 1 1 1.5
+#' # Result
+#' # ------
+#' # a c avg
+#' # 3 3 3.0
+#' # 1 1 1.5
#'
-#' Fits linear models on iris dataset by grouping on the 'Species' column and
-#' using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
-#' and 'Petal_Width' as training features.
+#' # Fits linear models on iris dataset by grouping on the 'Species' column and
+#' # using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
+#' # and 'Petal_Width' as training features.
#'
#' df <- createDataFrame (iris)
#' schema <- structType(structField("(Intercept)", "double"),
@@ -1734,12 +1746,12 @@ setMethod("dapplyCollect",
#' }, schema)
#' collect(df1)
#'
-#' Result
-#' ---------
-#' Model (Intercept) Sepal_Width Petal_Length Petal_Width
-#' 1 0.699883 0.3303370 0.9455356 -0.1697527
-#' 2 1.895540 0.3868576 0.9083370 -0.6792238
-#' 3 2.351890 0.6548350 0.2375602 0.2521257
+#' # Result
+#' # ---------
+#' # Model (Intercept) Sepal_Width Petal_Length Petal_Width
+#' # 1 0.699883 0.3303370 0.9455356 -0.1697527
+#' # 2 1.895540 0.3868576 0.9083370 -0.6792238
+#' # 3 2.351890 0.6548350 0.2375602 0.2521257
#'
#'}
#' @note gapply(SparkDataFrame) since 2.0.0
@@ -1757,20 +1769,30 @@ setMethod("gapply",
#'
#' @param cols grouping columns.
#' @param func a function to be applied to each group partition specified by grouping
-#' column of the SparkDataFrame. The function \code{func} takes as argument
-#' a key - grouping columns and a data frame - a local R data.frame.
-#' The output of \code{func} is a local R data.frame.
+#' column of the SparkDataFrame. See Details.
#' @return A data.frame.
#' @family SparkDataFrame functions
#' @aliases gapplyCollect,SparkDataFrame-method
#' @rdname gapplyCollect
#' @name gapplyCollect
+#' @details
+#' \code{func} is a function of two arguments. The first, usually named \code{key}
+#' (though this is not enforced) corresponds to the grouping key, will be an
+#' unnamed \code{list} of \code{length(cols)} length-one objects corresponding
+#' to the grouping columns' values for the current group.
+#'
+#' The second, herein \code{x}, will be a local \code{\link{data.frame}} with the
+#' columns of the input not in \code{cols} for the rows corresponding to \code{key}.
+#'
+#' The output of \code{func} must be a \code{data.frame} matching \code{schema} --
+#' in particular this means the names of the output \code{data.frame} are irrelevant
+#'
#' @seealso \link{gapply}
#' @examples
#'
#' \dontrun{
-#' Computes the arithmetic mean of the second column by grouping
-#' on the first and third columns. Output the grouping values and the average.
+#' # Computes the arithmetic mean of the second column by grouping
+#' # on the first and third columns. Output the grouping values and the average.
#'
#' df <- createDataFrame (
#' list(list(1L, 1, "1", 0.1), list(1L, 2, "1", 0.2), list(3L, 3, "3", 0.3)),
@@ -1785,8 +1807,8 @@ setMethod("gapply",
#' y
#' })
#'
-#' We can also group the data and afterwards call gapply on GroupedData.
-#' For Example:
+#' # We can also group the data and afterwards call gapply on GroupedData.
+#' # For example:
#' gdf <- group_by(df, "a", "c")
#' result <- gapplyCollect(
#' gdf,
@@ -1796,15 +1818,15 @@ setMethod("gapply",
#' y
#' })
#'
-#' Result
-#' ------
-#' key_a key_c mean_b
-#' 3 3 3.0
-#' 1 1 1.5
+#' # Result
+#' # ------
+#' # key_a key_c mean_b
+#' # 3 3 3.0
+#' # 1 1 1.5
#'
-#' Fits linear models on iris dataset by grouping on the 'Species' column and
-#' using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
-#' and 'Petal_Width' as training features.
+#' # Fits linear models on iris dataset by grouping on the 'Species' column and
+#' # using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
+#' # and 'Petal_Width' as training features.
#'
#' df <- createDataFrame (iris)
#' result <- gapplyCollect(
@@ -1816,12 +1838,12 @@ setMethod("gapply",
#' data.frame(t(coef(m)))
#' })
#'
-#' Result
-#'---------
-#' Model X.Intercept. Sepal_Width Petal_Length Petal_Width
-#' 1 0.699883 0.3303370 0.9455356 -0.1697527
-#' 2 1.895540 0.3868576 0.9083370 -0.6792238
-#' 3 2.351890 0.6548350 0.2375602 0.2521257
+#' # Result
+#' # ---------
+#' # Model X.Intercept. Sepal_Width Petal_Length Petal_Width
+#' # 1 0.699883 0.3303370 0.9455356 -0.1697527
+#' # 2 1.895540 0.3868576 0.9083370 -0.6792238
+#' # 3 2.351890 0.6548350 0.2375602 0.2521257
#'
#'}
#' @note gapplyCollect(SparkDataFrame) since 2.0.0
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org