You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/03/15 20:18:34 UTC
spark git commit: [SPARK-13896][SQL][STRING] Dataset.toJSON should
return Dataset
Repository: spark
Updated Branches:
refs/heads/master d89c71417 -> 50e3644d0
[SPARK-13896][SQL][STRING] Dataset.toJSON should return Dataset
## What changes were proposed in this pull request?
Change the return type of toJson in Dataset class
## How was this patch tested?
No additional unit test required.
Author: Stavros Kontopoulos <st...@typesafe.com>
Closes #11732 from skonto/fix_toJson.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/50e3644d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/50e3644d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/50e3644d
Branch: refs/heads/master
Commit: 50e3644d00c75de8273cce191debde68061e6e13
Parents: d89c714
Author: Stavros Kontopoulos <st...@typesafe.com>
Authored: Tue Mar 15 12:18:30 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Mar 15 12:18:30 2016 -0700
----------------------------------------------------------------------
sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala | 6 ++++--
.../spark/sql/execution/datasources/json/JsonSuite.scala | 4 ++--
.../apache/spark/sql/hive/MetastoreDataSourcesSuite.scala | 8 ++++----
3 files changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/50e3644d/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index f7ef0de..969fcdf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -1989,9 +1989,9 @@ class Dataset[T] private[sql](
* @group rdd
* @since 1.3.0
*/
- def toJSON: RDD[String] = {
+ def toJSON: Dataset[String] = {
val rowSchema = this.schema
- queryExecution.toRdd.mapPartitions { iter =>
+ val rdd = queryExecution.toRdd.mapPartitions { iter =>
val writer = new CharArrayWriter()
// create the Generator without separator inserted between 2 records
val gen = new JsonFactory().createGenerator(writer).setRootValueSeparator(null)
@@ -2013,6 +2013,8 @@ class Dataset[T] private[sql](
}
}
}
+ import sqlContext.implicits._
+ rdd.toDS
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/50e3644d/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 097ece3..4671b2d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -1096,7 +1096,7 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
assert(result2(3) === "{\"f1\":{\"f11\":4,\"f12\":true},\"f2\":{\"D4\":2147483644}}")
val jsonDF = sqlContext.read.json(primitiveFieldAndType)
- val primTable = sqlContext.read.json(jsonDF.toJSON)
+ val primTable = sqlContext.read.json(jsonDF.toJSON.rdd)
primTable.registerTempTable("primitiveTable")
checkAnswer(
sql("select * from primitiveTable"),
@@ -1109,7 +1109,7 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
)
val complexJsonDF = sqlContext.read.json(complexFieldAndType1)
- val compTable = sqlContext.read.json(complexJsonDF.toJSON)
+ val compTable = sqlContext.read.json(complexJsonDF.toJSON.rdd)
compTable.registerTempTable("complexTable")
// Access elements of a primitive array.
checkAnswer(
http://git-wip-us.apache.org/repos/asf/spark/blob/50e3644d/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index d7974f1..81420fe 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -164,7 +164,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
test("check change without refresh") {
withTempPath { tempDir =>
withTable("jsonTable") {
- (("a", "b") :: Nil).toDF().toJSON.saveAsTextFile(tempDir.getCanonicalPath)
+ (("a", "b") :: Nil).toDF().toJSON.rdd.saveAsTextFile(tempDir.getCanonicalPath)
sql(
s"""CREATE TABLE jsonTable
@@ -179,7 +179,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
Row("a", "b"))
Utils.deleteRecursively(tempDir)
- (("a1", "b1", "c1") :: Nil).toDF().toJSON.saveAsTextFile(tempDir.getCanonicalPath)
+ (("a1", "b1", "c1") :: Nil).toDF().toJSON.rdd.saveAsTextFile(tempDir.getCanonicalPath)
// Schema is cached so the new column does not show. The updated values in existing columns
// will show.
@@ -199,7 +199,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
test("drop, change, recreate") {
withTempPath { tempDir =>
- (("a", "b") :: Nil).toDF().toJSON.saveAsTextFile(tempDir.getCanonicalPath)
+ (("a", "b") :: Nil).toDF().toJSON.rdd.saveAsTextFile(tempDir.getCanonicalPath)
withTable("jsonTable") {
sql(
@@ -215,7 +215,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
Row("a", "b"))
Utils.deleteRecursively(tempDir)
- (("a", "b", "c") :: Nil).toDF().toJSON.saveAsTextFile(tempDir.getCanonicalPath)
+ (("a", "b", "c") :: Nil).toDF().toJSON.rdd.saveAsTextFile(tempDir.getCanonicalPath)
sql("DROP TABLE jsonTable")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org