You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/04/12 08:16:43 UTC
spark git commit: [MINOR][DOCS] JSON APIs related documentation fixes
Repository: spark
Updated Branches:
refs/heads/master b93843824 -> bca4259f1
[MINOR][DOCS] JSON APIs related documentation fixes
## What changes were proposed in this pull request?
This PR proposes corrections related to JSON APIs as below:
- Rendering links in Python documentation
- Replacing `RDD` to `Dataset` in programing guide
- Adding missing description about JSON Lines consistently in `DataFrameReader.json` in Python API
- De-duplicating little bit of `DataFrameReader.json` in Scala/Java API
## How was this patch tested?
Manually build the documentation via `jekyll build`. Corresponding snapstops will be left on the codes.
Note that currently there are Javadoc8 breaks in several places. These are proposed to be handled in https://github.com/apache/spark/pull/17477. So, this PR does not fix those.
Author: hyukjinkwon <gu...@gmail.com>
Closes #17602 from HyukjinKwon/minor-json-documentation.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bca4259f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bca4259f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bca4259f
Branch: refs/heads/master
Commit: bca4259f12b32eeb156b6755d0ec5e16d8e566b3
Parents: b938438
Author: hyukjinkwon <gu...@gmail.com>
Authored: Wed Apr 12 09:16:39 2017 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Wed Apr 12 09:16:39 2017 +0100
----------------------------------------------------------------------
docs/sql-programming-guide.md | 4 ++--
.../apache/spark/examples/sql/JavaSQLDataSourceExample.java | 2 +-
.../org/apache/spark/examples/sql/SQLDataSourceExample.scala | 2 +-
python/pyspark/sql/readwriter.py | 8 +++++---
python/pyspark/sql/streaming.py | 4 ++--
.../main/scala/org/apache/spark/sql/DataFrameReader.scala | 4 ++--
6 files changed, 13 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bca4259f/docs/sql-programming-guide.md
----------------------------------------------------------------------
diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md
index c425fac..28942b6 100644
--- a/docs/sql-programming-guide.md
+++ b/docs/sql-programming-guide.md
@@ -883,7 +883,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
<div data-lang="scala" markdown="1">
Spark SQL can automatically infer the schema of a JSON dataset and load it as a `Dataset[Row]`.
-This conversion can be done using `SparkSession.read.json()` on either an RDD of String,
+This conversion can be done using `SparkSession.read.json()` on either a `Dataset[String]`,
or a JSON file.
Note that the file that is offered as _a json file_ is not a typical JSON file. Each
@@ -897,7 +897,7 @@ For a regular multi-line JSON file, set the `wholeFile` option to `true`.
<div data-lang="java" markdown="1">
Spark SQL can automatically infer the schema of a JSON dataset and load it as a `Dataset<Row>`.
-This conversion can be done using `SparkSession.read().json()` on either an RDD of String,
+This conversion can be done using `SparkSession.read().json()` on either a `Dataset<String>`,
or a JSON file.
Note that the file that is offered as _a json file_ is not a typical JSON file. Each
http://git-wip-us.apache.org/repos/asf/spark/blob/bca4259f/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
index 1a70546..b66abae 100644
--- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
@@ -215,7 +215,7 @@ public class JavaSQLDataSourceExample {
// +------+
// Alternatively, a DataFrame can be created for a JSON dataset represented by
- // an Dataset[String] storing one JSON object per string.
+ // a Dataset<String> storing one JSON object per string.
List<String> jsonData = Arrays.asList(
"{\"name\":\"Yin\",\"address\":{\"city\":\"Columbus\",\"state\":\"Ohio\"}}");
Dataset<String> anotherPeopleDataset = spark.createDataset(jsonData, Encoders.STRING());
http://git-wip-us.apache.org/repos/asf/spark/blob/bca4259f/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
index 82fd56d..ad74da7 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
@@ -139,7 +139,7 @@ object SQLDataSourceExample {
// +------+
// Alternatively, a DataFrame can be created for a JSON dataset represented by
- // an Dataset[String] storing one JSON object per string
+ // a Dataset[String] storing one JSON object per string
val otherPeopleDataset = spark.createDataset(
"""{"name":"Yin","address":{"city":"Columbus","state":"Ohio"}}""" :: Nil)
val otherPeople = spark.read.json(otherPeopleDataset)
http://git-wip-us.apache.org/repos/asf/spark/blob/bca4259f/python/pyspark/sql/readwriter.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index d912f39..960fb88 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -173,8 +173,8 @@ class DataFrameReader(OptionUtils):
"""
Loads JSON files and returns the results as a :class:`DataFrame`.
- `JSON Lines <http://jsonlines.org/>`_(newline-delimited JSON) is supported by default.
- For JSON (one record per file), set the `wholeFile` parameter to ``true``.
+ `JSON Lines <http://jsonlines.org/>`_ (newline-delimited JSON) is supported by default.
+ For JSON (one record per file), set the ``wholeFile`` parameter to ``true``.
If the ``schema`` parameter is not specified, this function goes
through the input once to determine the input schema.
@@ -634,7 +634,9 @@ class DataFrameWriter(OptionUtils):
@since(1.4)
def json(self, path, mode=None, compression=None, dateFormat=None, timestampFormat=None):
- """Saves the content of the :class:`DataFrame` in JSON format at the specified path.
+ """Saves the content of the :class:`DataFrame` in JSON format
+ (`JSON Lines text format or newline-delimited JSON <http://jsonlines.org/>`_) at the
+ specified path.
:param path: the path in any Hadoop supported file system
:param mode: specifies the behavior of the save operation when data already exists.
http://git-wip-us.apache.org/repos/asf/spark/blob/bca4259f/python/pyspark/sql/streaming.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 3b60496..65b59d4 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -405,8 +405,8 @@ class DataStreamReader(OptionUtils):
"""
Loads a JSON file stream and returns the results as a :class:`DataFrame`.
- `JSON Lines <http://jsonlines.org/>`_(newline-delimited JSON) is supported by default.
- For JSON (one record per file), set the `wholeFile` parameter to ``true``.
+ `JSON Lines <http://jsonlines.org/>`_ (newline-delimited JSON) is supported by default.
+ For JSON (one record per file), set the ``wholeFile`` parameter to ``true``.
If the ``schema`` parameter is not specified, this function goes
through the input once to determine the input schema.
http://git-wip-us.apache.org/repos/asf/spark/blob/bca4259f/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 49691c1..c1b3291 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -268,8 +268,8 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads a JSON file (<a href="http://jsonlines.org/">JSON Lines text format or
- * newline-delimited JSON</a>) and returns the result as a `DataFrame`.
+ * Loads a JSON file and returns the results as a `DataFrame`.
+ *
* See the documentation on the overloaded `json()` method with varargs for more details.
*
* @since 1.4.0
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org