You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sh...@apache.org on 2016/01/22 19:35:05 UTC
spark git commit: [SPARK-12629][SPARKR] Fixes for DataFrame
saveAsTable method
Repository: spark
Updated Branches:
refs/heads/master e13c147e7 -> 8a88e1212
[SPARK-12629][SPARKR] Fixes for DataFrame saveAsTable method
I've tried to solve some of the issues mentioned in: https://issues.apache.org/jira/browse/SPARK-12629
Please, let me know what do you think.
Thanks!
Author: Narine Kokhlikyan <na...@gmail.com>
Closes #10580 from NarineK/sparkrSavaAsRable.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8a88e121
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8a88e121
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8a88e121
Branch: refs/heads/master
Commit: 8a88e121283472c26e70563a4e04c109e9b183b3
Parents: e13c147
Author: Narine Kokhlikyan <na...@gmail.com>
Authored: Fri Jan 22 10:35:02 2016 -0800
Committer: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Committed: Fri Jan 22 10:35:02 2016 -0800
----------------------------------------------------------------------
R/pkg/R/DataFrame.R | 23 +++++++++++++++++------
R/pkg/R/generics.R | 12 ++++++++++--
R/pkg/inst/tests/testthat/test_sparkSQL.R | 15 ++++++++++++++-
3 files changed, 41 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8a88e121/R/pkg/R/DataFrame.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 4653a73..3b7b825 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -1997,7 +1997,13 @@ setMethod("write.df",
signature(df = "DataFrame", path = "character"),
function(df, path, source = NULL, mode = "error", ...){
if (is.null(source)) {
- sqlContext <- get(".sparkRSQLsc", envir = .sparkREnv)
+ if (exists(".sparkRSQLsc", envir = .sparkREnv)) {
+ sqlContext <- get(".sparkRSQLsc", envir = .sparkREnv)
+ } else if (exists(".sparkRHivesc", envir = .sparkREnv)) {
+ sqlContext <- get(".sparkRHivesc", envir = .sparkREnv)
+ } else {
+ stop("sparkRHive or sparkRSQL context has to be specified")
+ }
source <- callJMethod(sqlContext, "getConf", "spark.sql.sources.default",
"org.apache.spark.sql.parquet")
}
@@ -2055,13 +2061,18 @@ setMethod("saveDF",
#' saveAsTable(df, "myfile")
#' }
setMethod("saveAsTable",
- signature(df = "DataFrame", tableName = "character", source = "character",
- mode = "character"),
+ signature(df = "DataFrame", tableName = "character"),
function(df, tableName, source = NULL, mode="error", ...){
if (is.null(source)) {
- sqlContext <- get(".sparkRSQLsc", envir = .sparkREnv)
- source <- callJMethod(sqlContext, "getConf", "spark.sql.sources.default",
- "org.apache.spark.sql.parquet")
+ if (exists(".sparkRSQLsc", envir = .sparkREnv)) {
+ sqlContext <- get(".sparkRSQLsc", envir = .sparkREnv)
+ } else if (exists(".sparkRHivesc", envir = .sparkREnv)) {
+ sqlContext <- get(".sparkRHivesc", envir = .sparkREnv)
+ } else {
+ stop("sparkRHive or sparkRSQL context has to be specified")
+ }
+ source <- callJMethod(sqlContext, "getConf", "spark.sql.sources.default",
+ "org.apache.spark.sql.parquet")
}
jmode <- convertToJSaveMode(mode)
options <- varargsToEnv(...)
http://git-wip-us.apache.org/repos/asf/spark/blob/8a88e121/R/pkg/R/generics.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R
index 9a8ab97..04784d5 100644
--- a/R/pkg/R/generics.R
+++ b/R/pkg/R/generics.R
@@ -539,7 +539,7 @@ setGeneric("sampleBy", function(x, col, fractions, seed) { standardGeneric("samp
#' @rdname saveAsTable
#' @export
-setGeneric("saveAsTable", function(df, tableName, source, mode, ...) {
+setGeneric("saveAsTable", function(df, tableName, source = NULL, mode = "error", ...) {
standardGeneric("saveAsTable")
})
@@ -552,7 +552,15 @@ setGeneric("transform", function(`_data`, ...) {standardGeneric("transform") })
#' @rdname write.df
#' @export
-setGeneric("saveDF", function(df, path, ...) { standardGeneric("saveDF") })
+setGeneric("write.df", function(df, path, source = NULL, mode = "error", ...) {
+ standardGeneric("write.df")
+})
+
+#' @rdname write.df
+#' @export
+setGeneric("saveDF", function(df, path, source = NULL, mode = "error", ...) {
+ standardGeneric("saveDF")
+})
#' @rdname write.json
#' @export
http://git-wip-us.apache.org/repos/asf/spark/blob/8a88e121/R/pkg/inst/tests/testthat/test_sparkSQL.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R
index e59841a..b52a11f 100644
--- a/R/pkg/inst/tests/testthat/test_sparkSQL.R
+++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R
@@ -953,8 +953,21 @@ test_that("test HiveContext", {
df3 <- sql(hiveCtx, "select * from json2")
expect_is(df3, "DataFrame")
expect_equal(count(df3), 3)
-
unlink(jsonPath2)
+
+ hivetestDataPath <- tempfile(pattern="sparkr-test", fileext=".tmp")
+ invisible(saveAsTable(df, "hivetestbl", path = hivetestDataPath))
+ df4 <- sql(hiveCtx, "select * from hivetestbl")
+ expect_is(df4, "DataFrame")
+ expect_equal(count(df4), 3)
+ unlink(hivetestDataPath)
+
+ parquetDataPath <- tempfile(pattern="sparkr-test", fileext=".tmp")
+ invisible(saveAsTable(df, "parquetest", "parquet", mode="overwrite", path=parquetDataPath))
+ df5 <- sql(hiveCtx, "select * from parquetest")
+ expect_is(df5, "DataFrame")
+ expect_equal(count(df5), 3)
+ unlink(parquetDataPath)
})
test_that("column operators", {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org