You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2020/05/19 02:02:51 UTC
[spark] branch branch-3.0 updated:
[SPARK-31739][PYSPARK][DOCS][MINOR] Fix docstring syntax issues and
misplaced space characters
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 18925c3 [SPARK-31739][PYSPARK][DOCS][MINOR] Fix docstring syntax issues and misplaced space characters
18925c3 is described below
commit 18925c34ffc403c1422dd8ac8ae731c213521387
Author: David Toneian <da...@toneian.com>
AuthorDate: Mon May 18 20:25:02 2020 +0900
[SPARK-31739][PYSPARK][DOCS][MINOR] Fix docstring syntax issues and misplaced space characters
This commit is published into the public domain.
Some syntax issues in docstrings have been fixed.
In some places, the documentation did not render as intended, e.g. parameter documentations were not formatted as such.
Slight improvements in documentation.
Manual testing and `dev/lint-python` run. No new Sphinx warnings arise due to this change.
Closes #28559 from DavidToneian/SPARK-31739.
Authored-by: David Toneian <da...@toneian.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
python/pyspark/ml/clustering.py | 2 +-
python/pyspark/ml/regression.py | 2 +-
python/pyspark/ml/util.py | 1 +
python/pyspark/mllib/util.py | 4 ++--
python/pyspark/sql/dataframe.py | 6 +++---
python/pyspark/sql/readwriter.py | 14 +++++++-------
python/pyspark/sql/streaming.py | 26 +++++++++++++-------------
7 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index 7465cef..e1c2732 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -795,7 +795,7 @@ class BisectingKMeansModel(JavaModel, _BisectingKMeansParams, JavaMLWritable, Ja
Computes the sum of squared distances between the input points
and their corresponding cluster centers.
- ..note:: Deprecated in 3.0.0. It will be removed in future versions. Use
+ .. note:: Deprecated in 3.0.0. It will be removed in future versions. Use
ClusteringEvaluator instead. You can also get the cost on the training dataset in the
summary.
"""
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index a4c9782..f367bb8 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -53,7 +53,7 @@ class JavaRegressor(JavaPredictor, _JavaPredictorParams):
class JavaRegressionModel(JavaPredictionModel, _JavaPredictorParams):
"""
Java Model produced by a ``_JavaRegressor``.
- To be mixed in with class:`pyspark.ml.JavaModel`
+ To be mixed in with :class:`pyspark.ml.JavaModel`
.. versionadded:: 3.0.0
"""
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index 35ad551..aac2b38 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -563,6 +563,7 @@ class DefaultParamsReader(MLReader):
class HasTrainingSummary(object):
"""
Base class for models that provides Training summary.
+
.. versionadded:: 3.0.0
"""
diff --git a/python/pyspark/mllib/util.py b/python/pyspark/mllib/util.py
index 1a0ce42..f0f9cda 100644
--- a/python/pyspark/mllib/util.py
+++ b/python/pyspark/mllib/util.py
@@ -372,7 +372,7 @@ class Saveable(object):
* human-readable (JSON) model metadata to path/metadata/
* Parquet formatted data to path/data/
- The model may be loaded using py:meth:`Loader.load`.
+ The model may be loaded using :py:meth:`Loader.load`.
:param sc: Spark context used to save model data.
:param path: Path specifying the directory in which to save
@@ -412,7 +412,7 @@ class Loader(object):
def load(cls, sc, path):
"""
Load a model from the given path. The model should have been
- saved using py:meth:`Saveable.save`.
+ saved using :py:meth:`Saveable.save`.
:param sc: Spark context used for loading model files.
:param path: Path specifying the directory to which the model
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 2a366dc..6f4fdd3 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -2138,7 +2138,7 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
@ignore_unicode_prefix
def toDF(self, *cols):
- """Returns a new class:`DataFrame` that with new specified column names
+ """Returns a new :class:`DataFrame` that with new specified column names
:param cols: list of new column names (string)
@@ -2150,9 +2150,9 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
@since(3.0)
def transform(self, func):
- """Returns a new class:`DataFrame`. Concise syntax for chaining custom transformations.
+ """Returns a new :class:`DataFrame`. Concise syntax for chaining custom transformations.
- :param func: a function that takes and returns a class:`DataFrame`.
+ :param func: a function that takes and returns a :class:`DataFrame`.
>>> from pyspark.sql.functions import col
>>> df = spark.createDataFrame([(1, 1.0), (2, 2.0)], ["int", "float"])
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 6ad6377..336345e 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -223,15 +223,15 @@ class DataFrameReader(OptionUtils):
:param mode: allows a mode for dealing with corrupt records during parsing. If None is
set, it uses the default value, ``PERMISSIVE``.
- * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+ * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
fields to ``null``. To keep corrupt records, an user can set a string type \
field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
schema does not have the field, it drops corrupt records during parsing. \
When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
field in an output schema.
- * ``DROPMALFORMED`` : ignores the whole corrupted records.
- * ``FAILFAST`` : throws an exception when it meets corrupted records.
+ * ``DROPMALFORMED``: ignores the whole corrupted records.
+ * ``FAILFAST``: throws an exception when it meets corrupted records.
:param columnNameOfCorruptRecord: allows renaming the new field having malformed string
created by ``PERMISSIVE`` mode. This overrides
@@ -470,7 +470,7 @@ class DataFrameReader(OptionUtils):
be controlled by ``spark.sql.csv.parser.columnPruning.enabled``
(enabled by default).
- * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+ * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
fields to ``null``. To keep corrupt records, an user can set a string type \
field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
@@ -479,8 +479,8 @@ class DataFrameReader(OptionUtils):
When it meets a record having fewer tokens than the length of the schema, \
sets ``null`` to extra fields. When the record has more tokens than the \
length of the schema, it drops extra tokens.
- * ``DROPMALFORMED`` : ignores the whole corrupted records.
- * ``FAILFAST`` : throws an exception when it meets corrupted records.
+ * ``DROPMALFORMED``: ignores the whole corrupted records.
+ * ``FAILFAST``: throws an exception when it meets corrupted records.
:param columnNameOfCorruptRecord: allows renaming the new field having malformed string
created by ``PERMISSIVE`` mode. This overrides
@@ -830,7 +830,7 @@ class DataFrameWriter(OptionUtils):
def insertInto(self, tableName, overwrite=None):
"""Inserts the content of the :class:`DataFrame` to the specified table.
- It requires that the schema of the class:`DataFrame` is the same as the
+ It requires that the schema of the :class:`DataFrame` is the same as the
schema of the table.
Optionally overwriting any existing data.
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 05cf331..2450a4c 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -461,15 +461,15 @@ class DataStreamReader(OptionUtils):
:param mode: allows a mode for dealing with corrupt records during parsing. If None is
set, it uses the default value, ``PERMISSIVE``.
- * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+ * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
fields to ``null``. To keep corrupt records, an user can set a string type \
field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
schema does not have the field, it drops corrupt records during parsing. \
When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
field in an output schema.
- * ``DROPMALFORMED`` : ignores the whole corrupted records.
- * ``FAILFAST`` : throws an exception when it meets corrupted records.
+ * ``DROPMALFORMED``: ignores the whole corrupted records.
+ * ``FAILFAST``: throws an exception when it meets corrupted records.
:param columnNameOfCorruptRecord: allows renaming the new field having malformed string
created by ``PERMISSIVE`` mode. This overrides
@@ -707,7 +707,7 @@ class DataStreamReader(OptionUtils):
:param mode: allows a mode for dealing with corrupt records during parsing. If None is
set, it uses the default value, ``PERMISSIVE``.
- * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+ * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
fields to ``null``. To keep corrupt records, an user can set a string type \
field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
@@ -716,8 +716,8 @@ class DataStreamReader(OptionUtils):
When it meets a record having fewer tokens than the length of the schema, \
sets ``null`` to extra fields. When the record has more tokens than the \
length of the schema, it drops extra tokens.
- * ``DROPMALFORMED`` : ignores the whole corrupted records.
- * ``FAILFAST`` : throws an exception when it meets corrupted records.
+ * ``DROPMALFORMED``: ignores the whole corrupted records.
+ * ``FAILFAST``: throws an exception when it meets corrupted records.
:param columnNameOfCorruptRecord: allows renaming the new field having malformed string
created by ``PERMISSIVE`` mode. This overrides
@@ -795,11 +795,11 @@ class DataStreamWriter(object):
Options include:
- * `append`:Only the new rows in the streaming DataFrame/Dataset will be written to
+ * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to
the sink
- * `complete`:All the rows in the streaming DataFrame/Dataset will be written to the sink
+ * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the sink
every time these is some updates
- * `update`:only the rows that were updated in the streaming DataFrame/Dataset will be
+ * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
written to the sink every time there are some updates. If the query doesn't contain
aggregations, it will be equivalent to `append` mode.
@@ -1170,11 +1170,11 @@ class DataStreamWriter(object):
:param outputMode: specifies how data of a streaming DataFrame/Dataset is written to a
streaming sink.
- * `append`:Only the new rows in the streaming DataFrame/Dataset will be written to the
+ * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the
sink
- * `complete`:All the rows in the streaming DataFrame/Dataset will be written to the sink
- every time these is some updates
- * `update`:only the rows that were updated in the streaming DataFrame/Dataset will be
+ * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the
+ sink every time these is some updates
+ * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
written to the sink every time there are some updates. If the query doesn't contain
aggregations, it will be equivalent to `append` mode.
:param partitionBy: names of partitioning columns
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org