You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sh...@apache.org on 2016/06/17 22:56:07 UTC
spark git commit: [SPARK-15925][SPARKR] R DataFrame add back
registerTempTable, add tests
Repository: spark
Updated Branches:
refs/heads/master 1a65e62a7 -> ef3cc4fc0
[SPARK-15925][SPARKR] R DataFrame add back registerTempTable, add tests
## What changes were proposed in this pull request?
Add registerTempTable to DataFrame with Deprecate
## How was this patch tested?
unit tests
shivaram liancheng
Author: Felix Cheung <fe...@hotmail.com>
Closes #13722 from felixcheung/rregistertemptable.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ef3cc4fc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ef3cc4fc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ef3cc4fc
Branch: refs/heads/master
Commit: ef3cc4fc096e831823d62af4fd2a12ae88d434b4
Parents: 1a65e62
Author: Felix Cheung <fe...@hotmail.com>
Authored: Fri Jun 17 15:56:03 2016 -0700
Committer: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Committed: Fri Jun 17 15:56:03 2016 -0700
----------------------------------------------------------------------
R/pkg/NAMESPACE | 1 +
R/pkg/R/DataFrame.R | 30 ++++++++++++++++++++++++--
R/pkg/R/generics.R | 14 +++++++-----
R/pkg/inst/tests/testthat/test_sparkSQL.R | 30 ++++++++++++++++----------
4 files changed, 57 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ef3cc4fc/R/pkg/NAMESPACE
----------------------------------------------------------------------
diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE
index 8db4d5c..5db43ae 100644
--- a/R/pkg/NAMESPACE
+++ b/R/pkg/NAMESPACE
@@ -82,6 +82,7 @@ exportMethods("arrange",
"persist",
"printSchema",
"rbind",
+ "registerTempTable",
"rename",
"repartition",
"sample",
http://git-wip-us.apache.org/repos/asf/spark/blob/ef3cc4fc/R/pkg/R/DataFrame.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index c710bff..231e4f0 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -457,6 +457,32 @@ setMethod("createOrReplaceTempView",
invisible(callJMethod(x@sdf, "createOrReplaceTempView", viewName))
})
+#' (Deprecated) Register Temporary Table
+#' Registers a SparkDataFrame as a Temporary Table in the SQLContext
+#' @param x A SparkDataFrame
+#' @param tableName A character vector containing the name of the table
+#'
+#' @family SparkDataFrame functions
+#' @seealso \link{createOrReplaceTempView}
+#' @rdname registerTempTable-deprecated
+#' @name registerTempTable
+#' @export
+#' @examples
+#'\dontrun{
+#' sc <- sparkR.init()
+#' sqlContext <- sparkRSQL.init(sc)
+#' path <- "path/to/file.json"
+#' df <- read.json(path)
+#' registerTempTable(df, "json_df")
+#' new_df <- sql("SELECT * FROM json_df")
+#'}
+setMethod("registerTempTable",
+ signature(x = "SparkDataFrame", tableName = "character"),
+ function(x, tableName) {
+ .Deprecated("createOrReplaceTempView")
+ invisible(callJMethod(x@sdf, "createOrReplaceTempView", tableName))
+ })
+
#' insertInto
#'
#' Insert the contents of a SparkDataFrame into a table registered in the current SQL Context.
@@ -1286,7 +1312,7 @@ setMethod("dapplyCollect",
#' @name gapply
#' @export
#' @examples
-#'
+#'
#' \dontrun{
#' Computes the arithmetic mean of the second column by grouping
#' on the first and third columns. Output the grouping values and the average.
@@ -1317,7 +1343,7 @@ setMethod("dapplyCollect",
#' Fits linear models on iris dataset by grouping on the 'Species' column and
#' using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
#' and 'Petal_Width' as training features.
-#'
+#'
#' df <- createDataFrame (iris)
#' schema <- structType(structField("(Intercept)", "double"),
#' structField("Sepal_Width", "double"),structField("Petal_Length", "double"),
http://git-wip-us.apache.org/repos/asf/spark/blob/ef3cc4fc/R/pkg/R/generics.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R
index 8164e77..594bf2e 100644
--- a/R/pkg/R/generics.R
+++ b/R/pkg/R/generics.R
@@ -446,6 +446,13 @@ setGeneric("covar_samp", function(col1, col2) {standardGeneric("covar_samp") })
#' @export
setGeneric("covar_pop", function(col1, col2) {standardGeneric("covar_pop") })
+#' @rdname createOrReplaceTempView
+#' @export
+setGeneric("createOrReplaceTempView",
+ function(x, viewName) {
+ standardGeneric("createOrReplaceTempView")
+ })
+
#' @rdname dapply
#' @export
setGeneric("dapply", function(x, func, schema) { standardGeneric("dapply") })
@@ -548,12 +555,9 @@ setGeneric("printSchema", function(x) { standardGeneric("printSchema") })
#' @export
setGeneric("rename", function(x, ...) { standardGeneric("rename") })
-#' @rdname createOrReplaceTempView
+#' @rdname registerTempTable-deprecated
#' @export
-setGeneric("createOrReplaceTempView",
- function(x, viewName) {
- standardGeneric("createOrReplaceTempView")
- })
+setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })
#' @rdname sample
#' @export
http://git-wip-us.apache.org/repos/asf/spark/blob/ef3cc4fc/R/pkg/inst/tests/testthat/test_sparkSQL.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R
index 11d6936..7aa03a9 100644
--- a/R/pkg/inst/tests/testthat/test_sparkSQL.R
+++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R
@@ -443,22 +443,21 @@ test_that("jsonRDD() on a RDD with json string", {
expect_equal(count(df), 6)
})
-test_that("test cache, uncache and clearCache", {
- df <- read.json(jsonPath)
- createOrReplaceTempView(df, "table1")
- cacheTable("table1")
- uncacheTable("table1")
- clearCache()
- dropTempTable("table1")
-})
-
test_that("test tableNames and tables", {
df <- read.json(jsonPath)
createOrReplaceTempView(df, "table1")
expect_equal(length(tableNames()), 1)
- df <- tables()
- expect_equal(count(df), 1)
+ tables <- tables()
+ expect_equal(count(tables), 1)
+
+ suppressWarnings(registerTempTable(df, "table2"))
+ tables <- tables()
+ expect_equal(count(tables), 2)
dropTempTable("table1")
+ dropTempTable("table2")
+
+ tables <- tables()
+ expect_equal(count(tables), 0)
})
test_that(
@@ -471,6 +470,15 @@ test_that(
dropTempTable("table1")
})
+test_that("test cache, uncache and clearCache", {
+ df <- read.json(jsonPath)
+ createOrReplaceTempView(df, "table1")
+ cacheTable("table1")
+ uncacheTable("table1")
+ clearCache()
+ dropTempTable("table1")
+})
+
test_that("insertInto() on a registered table", {
df <- read.df(jsonPath, "json")
write.df(df, parquetPath, "parquet", "overwrite")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org