You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2021/02/02 00:31:32 UTC
[spark] branch branch-3.1 updated:
[SPARK-34300][PYSPARK][DOCS][MINOR] Fix some typos and syntax issues in
docstrings and output of `dev/lint-python`
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 6831308 [SPARK-34300][PYSPARK][DOCS][MINOR] Fix some typos and syntax issues in docstrings and output of `dev/lint-python`
6831308 is described below
commit 6831308ddcfa97c4667218641602de89c4c689b6
Author: David Toneian <da...@toneian.com>
AuthorDate: Tue Feb 2 09:30:50 2021 +0900
[SPARK-34300][PYSPARK][DOCS][MINOR] Fix some typos and syntax issues in docstrings and output of `dev/lint-python`
This changeset is published into the public domain.
### What changes were proposed in this pull request?
Some typos and syntax issues in docstrings and the output of `dev/lint-python` have been fixed.
### Why are the changes needed?
In some places, the documentation did not refer to parameters or classes by the full and correct name, potentially causing uncertainty in the reader or rendering issues in Sphinx. Also, a typo in the standard output of `dev/lint-python` was fixed.
### Does this PR introduce _any_ user-facing change?
Slight improvements in documentation, and in standard output of `dev/lint-python`.
### How was this patch tested?
Manual testing and `dev/lint-python` run. No new Sphinx warnings arise due to this change.
Closes #31401 from DavidToneian/SPARK-34300.
Authored-by: David Toneian <da...@toneian.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
(cherry picked from commit d99d0d27be875bba692bcfe376f90c930e170380)
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
dev/lint-python | 2 +-
python/pyspark/sql/avro/functions.py | 4 +-
python/pyspark/sql/functions.py | 285 ++++++++++++++++++-----------------
3 files changed, 147 insertions(+), 144 deletions(-)
diff --git a/dev/lint-python b/dev/lint-python
index 2c244e0..9de4c28 100755
--- a/dev/lint-python
+++ b/dev/lint-python
@@ -208,7 +208,7 @@ function sphinx_test {
# See also https://github.com/sphinx-doc/sphinx/issues/7551.
PYTHON_HAS_SPHINX_3_0=$("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion; import sphinx; print(LooseVersion(sphinx.__version__) < LooseVersion("3.1.0"))')
if [[ "$PYTHON_HAS_SPHINX_3_0" == "False" ]]; then
- echo "$PYTHON_EXECUTABLE has Sphinx 3.1+ installed but it requires lower then 3.1. Skipping Sphinx build for now."
+ echo "$PYTHON_EXECUTABLE has Sphinx 3.1+ installed but it requires lower than 3.1. Skipping Sphinx build for now."
echo
return
fi
diff --git a/python/pyspark/sql/avro/functions.py b/python/pyspark/sql/avro/functions.py
index ce32281..2480bf8 100644
--- a/python/pyspark/sql/avro/functions.py
+++ b/python/pyspark/sql/avro/functions.py
@@ -37,7 +37,7 @@ def from_avro(data, jsonFormatSchema, options={}):
Parameters
----------
- data : :class:`Column` or str
+ data : :class:`~pyspark.sql.Column` or str
the binary column.
jsonFormatSchema : str
the avro schema in JSON string format.
@@ -86,7 +86,7 @@ def to_avro(data, jsonFormatSchema=""):
Parameters
----------
- data : :class:`Column` or str
+ data : :class:`~pyspark.sql.Column` or str
the data column.
jsonFormatSchema : str, optional
user-specified output avro schema in JSON string format.
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 90f2a45..51ab9c1 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -52,7 +52,7 @@ def _get_get_jvm_function(name, sc):
def _invoke_function(name, *args):
"""
Invokes JVM function identified by name with args
- and wraps the result with :class:`Column`.
+ and wraps the result with :class:`~pyspark.sql.Column`.
"""
jf = _get_get_jvm_function(name, SparkContext._active_spark_context)
return Column(jf(*args))
@@ -61,7 +61,7 @@ def _invoke_function(name, *args):
def _invoke_function_over_column(name, col):
"""
Invokes unary JVM function identified by name
- and wraps the result with :class:`Column`.
+ and wraps the result with :class:`~pyspark.sql.Column`.
"""
return _invoke_function(name, _to_java_column(col))
@@ -69,7 +69,7 @@ def _invoke_function_over_column(name, col):
def _invoke_binary_math_function(name, col1, col2):
"""
Invokes binary JVM math function identified by name
- and wraps the result with :class:`Column`.
+ and wraps the result with :class:`~pyspark.sql.Column`.
"""
return _invoke_function(
name,
@@ -86,7 +86,7 @@ def _options_to_str(options):
def lit(col):
"""
- Creates a :class:`Column` of literal value.
+ Creates a :class:`~pyspark.sql.Column` of literal value.
.. versionadded:: 1.3.0
@@ -101,7 +101,7 @@ def lit(col):
@since(1.3)
def col(col):
"""
- Returns a :class:`Column` based on the given column name.'
+ Returns a :class:`~pyspark.sql.Column` based on the given column name.'
"""
return _invoke_function("col", col)
@@ -109,7 +109,7 @@ def col(col):
@since(1.3)
def column(col):
"""
- Returns a :class:`Column` based on the given column name.'
+ Returns a :class:`~pyspark.sql.Column` based on the given column name.'
"""
return col(col)
@@ -214,7 +214,7 @@ def acos(col):
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
inverse cosine of `col`, as if computed by `java.lang.Math.acos()`
"""
return _invoke_function_over_column("acos", col)
@@ -228,7 +228,7 @@ def acosh(col):
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
"""
return _invoke_function_over_column("acosh", col)
@@ -240,7 +240,7 @@ def asin(col):
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
inverse sine of `col`, as if computed by `java.lang.Math.asin()`
"""
return _invoke_function_over_column("asin", col)
@@ -254,7 +254,7 @@ def asinh(col):
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
"""
return _invoke_function_over_column("asinh", col)
@@ -265,7 +265,7 @@ def atan(col):
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
inverse tangent of `col`, as if computed by `java.lang.Math.atan()`
"""
return _invoke_function_over_column("atan", col)
@@ -279,7 +279,7 @@ def atanh(col):
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
"""
return _invoke_function_over_column("atanh", col)
@@ -306,12 +306,12 @@ def cos(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
angle in radians
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
cosine of the angle, as if computed by `java.lang.Math.cos()`.
"""
return _invoke_function_over_column("cos", col)
@@ -323,12 +323,12 @@ def cosh(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
hyperbolic angle
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh()`
"""
return _invoke_function_over_column("cosh", col)
@@ -405,11 +405,11 @@ def sin(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
sine of the angle, as if computed by `java.lang.Math.sin()`
"""
return _invoke_function_over_column("sin", col)
@@ -421,12 +421,12 @@ def sinh(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
hyperbolic angle
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
hyperbolic sine of the given value,
as if computed by `java.lang.Math.sinh()`
"""
@@ -439,12 +439,12 @@ def tan(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
angle in radians
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
tangent of the given value, as if computed by `java.lang.Math.tan()`
"""
return _invoke_function_over_column("tan", col)
@@ -456,12 +456,12 @@ def tanh(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
hyperbolic angle
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
hyperbolic tangent of the given value
as if computed by `java.lang.Math.tanh()`
"""
@@ -660,12 +660,12 @@ def degrees(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
angle in radians
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
angle in degrees, as if computed by `java.lang.Math.toDegrees()`
"""
return _invoke_function_over_column("degrees", col)
@@ -680,12 +680,12 @@ def radians(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
angle in degrees
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
angle in radians, as if computed by `java.lang.Math.toRadians()`
"""
return _invoke_function_over_column("radians", col)
@@ -697,14 +697,14 @@ def atan2(col1, col2):
Parameters
----------
- col1 : str, :class:`Column` or float
+ col1 : str, :class:`~pyspark.sql.Column` or float
coordinate on y-axis
- col2 : str, :class:`Column` or float
+ col2 : str, :class:`~pyspark.sql.Column` or float
coordinate on x-axis
Returns
-------
- :class:`Column`
+ :class:`~pyspark.sql.Column`
the `theta` component of the point
(`r`, `theta`)
in polar coordinates that corresponds to the point
@@ -798,14 +798,14 @@ def approxCountDistinct(col, rsd=None):
def approx_count_distinct(col, rsd=None):
- """Aggregate function: returns a new :class:`Column` for approximate distinct count of
- column `col`.
+ """Aggregate function: returns a new :class:`~pyspark.sql.Column` for approximate distinct count
+ of column `col`.
.. versionadded:: 2.1.0
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
rsd : float, optional
maximum relative standard deviation allowed (default = 0.05).
For rsd < 0.01, it is more efficient to use :func:`countDistinct`
@@ -872,8 +872,8 @@ def coalesce(*cols):
def corr(col1, col2):
- """Returns a new :class:`Column` for the Pearson Correlation Coefficient for ``col1``
- and ``col2``.
+ """Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for
+ ``col1`` and ``col2``.
.. versionadded:: 1.6.0
@@ -890,7 +890,8 @@ def corr(col1, col2):
def covar_pop(col1, col2):
- """Returns a new :class:`Column` for the population covariance of ``col1`` and ``col2``.
+ """Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and
+ ``col2``.
.. versionadded:: 2.0.0
@@ -907,7 +908,8 @@ def covar_pop(col1, col2):
def covar_samp(col1, col2):
- """Returns a new :class:`Column` for the sample covariance of ``col1`` and ``col2``.
+ """Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and
+ ``col2``.
.. versionadded:: 2.0.0
@@ -924,7 +926,7 @@ def covar_samp(col1, col2):
def countDistinct(col, *cols):
- """Returns a new :class:`Column` for distinct count of ``col`` or ``cols``.
+ """Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``.
.. versionadded:: 1.3.0
@@ -1332,8 +1334,8 @@ def struct(*cols):
Parameters
----------
- cols : list, set, str or :class:`Column`
- column names or :class:`Column`\\s to contain in the output struct.
+ cols : list, set, str or :class:`~pyspark.sql.Column`
+ column names or :class:`~pyspark.sql.Column`\\s to contain in the output struct.
Examples
--------
@@ -1389,16 +1391,17 @@ def least(*cols):
def when(condition, value):
"""Evaluates a list of conditions and returns one of multiple possible result expressions.
- If :func:`Column.otherwise` is not invoked, None is returned for unmatched conditions.
+ If :func:`pyspark.sql.Column.otherwise` is not invoked, None is returned for unmatched
+ conditions.
.. versionadded:: 1.4.0
Parameters
----------
- condition : :class:`Column`
- a boolean :class:`Column` expression.
+ condition : :class:`~pyspark.sql.Column`
+ a boolean :class:`~pyspark.sql.Column` expression.
value :
- a literal value, or a :class:`Column` expression.
+ a literal value, or a :class:`~pyspark.sql.Column` expression.
>>> df.select(when(df['age'] == 2, 3).otherwise(4).alias("age")).collect()
[Row(age=3), Row(age=4)]
@@ -1488,7 +1491,7 @@ def factorial(col):
def lag(col, offset=1, default=None):
"""
Window function: returns the value that is `offset` rows before the current row, and
- `defaultValue` if there is less than `offset` rows before the current row. For example,
+ `default` if there is less than `offset` rows before the current row. For example,
an `offset` of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.
@@ -1497,7 +1500,7 @@ def lag(col, offset=1, default=None):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
offset : int, optional
number of row to extend
@@ -1511,7 +1514,7 @@ def lag(col, offset=1, default=None):
def lead(col, offset=1, default=None):
"""
Window function: returns the value that is `offset` rows after the current row, and
- `defaultValue` if there is less than `offset` rows after the current row. For example,
+ `default` if there is less than `offset` rows after the current row. For example,
an `offset` of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.
@@ -1520,7 +1523,7 @@ def lead(col, offset=1, default=None):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
offset : int, optional
number of row to extend
@@ -1545,7 +1548,7 @@ def nth_value(col, offset, ignoreNulls=False):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
offset : int, optional
number of row to use as the value
@@ -1872,7 +1875,7 @@ def months_between(date1, date2, roundOff=True):
def to_date(col, format=None):
- """Converts a :class:`Column` into :class:`pyspark.sql.types.DateType`
+ """Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.DateType`
using the optionally specified format. Specify formats according to `datetime pattern`_.
By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format
is omitted. Equivalent to ``col.cast("date")``.
@@ -1900,7 +1903,7 @@ def to_date(col, format=None):
def to_timestamp(col, format=None):
- """Converts a :class:`Column` into :class:`pyspark.sql.types.TimestampType`
+ """Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.TimestampType`
using the optionally specified format. Specify formats according to `datetime pattern`_.
By default, it follows casting rules to :class:`pyspark.sql.types.TimestampType` if the format
is omitted. Equivalent to ``col.cast("timestamp")``.
@@ -1935,7 +1938,7 @@ def trunc(date, format):
Parameters
----------
- date : :class:`Column` or str
+ date : :class:`~pyspark.sql.Column` or str
format : str
'year', 'yyyy', 'yy' or 'month', 'mon', 'mm'
@@ -1962,7 +1965,7 @@ def date_trunc(format, timestamp):
format : str
'year', 'yyyy', 'yy', 'month', 'mon', 'mm',
'day', 'dd', 'hour', 'minute', 'second', 'week', 'quarter'
- timestamp : :class:`Column` or str
+ timestamp : :class:`~pyspark.sql.Column` or str
Examples
--------
@@ -2074,9 +2077,9 @@ def from_utc_timestamp(timestamp, tz):
Parameters
----------
- timestamp : :class:`Column` or str
+ timestamp : :class:`~pyspark.sql.Column` or str
the column that contains timestamps
- tz : :class:`Column` or str
+ tz : :class:`~pyspark.sql.Column` or str
A string detailing the time zone ID that the input should be adjusted to. It should
be in the format of either region-based zone IDs or zone offsets. Region IDs must
have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
@@ -2085,7 +2088,7 @@ def from_utc_timestamp(timestamp, tz):
because they can be ambiguous.
.. versionchanged:: 2.4
- `tz` can take a :class:`Column` containing timezone ID strings.
+ `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings.
Examples
--------
@@ -2120,9 +2123,9 @@ def to_utc_timestamp(timestamp, tz):
Parameters
----------
- timestamp : :class:`Column` or str
+ timestamp : :class:`~pyspark.sql.Column` or str
the column that contains timestamps
- tz : :class:`Column` or str
+ tz : :class:`~pyspark.sql.Column` or str
A string detailing the time zone ID that the input should be adjusted to. It should
be in the format of either region-based zone IDs or zone offsets. Region IDs must
have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
@@ -2131,7 +2134,7 @@ def to_utc_timestamp(timestamp, tz):
because they can be ambiguous.
.. versionchanged:: 2.4.0
- `tz` can take a :class:`Column` containing timezone ID strings.
+ `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings.
Examples
--------
@@ -2485,7 +2488,7 @@ def format_number(col, d):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
the column name of the numeric value to be formatted
d : int
the N decimal places
@@ -2507,8 +2510,8 @@ def format_string(format, *cols):
----------
format : str
string that can contain embedded format tags and used as result column's value
- cols : :class:`Column` or str
- column names or :class:`Column`\\s to be used in formatting
+ cols : :class:`~pyspark.sql.Column` or str
+ column names or :class:`~pyspark.sql.Column`\\s to be used in formatting
Examples
--------
@@ -2646,7 +2649,7 @@ def locate(substr, str, pos=1):
----------
substr : str
a string
- str : :class:`Column` or str
+ str : :class:`~pyspark.sql.Column` or str
a Column of :class:`pyspark.sql.types.StringType`
pos : int, optional
start position (zero based)
@@ -2722,7 +2725,7 @@ def split(str, pattern, limit=-1):
Parameters
----------
- str : :class:`Column` or str
+ str : :class:`~pyspark.sql.Column` or str
a string expression to split
pattern : str
a string representing a regular expression. The regex string should be
@@ -2910,8 +2913,8 @@ def create_map(*cols):
Parameters
----------
- cols : :class:`Column` or str
- column names or :class:`Column`\\s that are
+ cols : :class:`~pyspark.sql.Column` or str
+ column names or :class:`~pyspark.sql.Column`\\s that are
grouped as key-value pairs, e.g. (key1, value1, key2, value2, ...).
Examples
@@ -2935,9 +2938,9 @@ def map_from_arrays(col1, col2):
Parameters
----------
- col1 : :class:`Column` or str
+ col1 : :class:`~pyspark.sql.Column` or str
name of column containing a set of keys. All elements should not be null
- col2 : :class:`Column` or str
+ col2 : :class:`~pyspark.sql.Column` or str
name of column containing a set of values
Examples
@@ -2961,8 +2964,8 @@ def array(*cols):
Parameters
----------
- cols : :class:`Column` or str
- column names or :class:`Column`\\s that have
+ cols : :class:`~pyspark.sql.Column` or str
+ column names or :class:`~pyspark.sql.Column`\\s that have
the same data type.
Examples
@@ -2988,7 +2991,7 @@ def array_contains(col, value):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column containing array
value :
value or column to check for in array
@@ -3033,11 +3036,11 @@ def slice(x, start, length):
Parameters
----------
- x : :class:`Column` or str
+ x : :class:`~pyspark.sql.Column` or str
the array to be sliced
- start : :class:`Column` or int
+ start : :class:`~pyspark.sql.Column` or int
the starting index
- length : :class:`Column` or int
+ length : :class:`~pyspark.sql.Column` or int
the length of the slice
Examples
@@ -3129,7 +3132,7 @@ def element_at(col, extraction):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column containing array or map
extraction :
index to check for in array or key to check for in map
@@ -3161,7 +3164,7 @@ def array_remove(col, element):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column containing array
element :
element to be removed from the array
@@ -3184,7 +3187,7 @@ def array_distinct(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3206,9 +3209,9 @@ def array_intersect(col1, col2):
Parameters
----------
- col1 : :class:`Column` or str
+ col1 : :class:`~pyspark.sql.Column` or str
name of column containing array
- col2 : :class:`Column` or str
+ col2 : :class:`~pyspark.sql.Column` or str
name of column containing array
Examples
@@ -3231,9 +3234,9 @@ def array_union(col1, col2):
Parameters
----------
- col1 : :class:`Column` or str
+ col1 : :class:`~pyspark.sql.Column` or str
name of column containing array
- col2 : :class:`Column` or str
+ col2 : :class:`~pyspark.sql.Column` or str
name of column containing array
Examples
@@ -3256,9 +3259,9 @@ def array_except(col1, col2):
Parameters
----------
- col1 : :class:`Column` or str
+ col1 : :class:`~pyspark.sql.Column` or str
name of column containing array
- col2 : :class:`Column` or str
+ col2 : :class:`~pyspark.sql.Column` or str
name of column containing array
Examples
@@ -3412,7 +3415,7 @@ def get_json_object(col, path):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
string column in json format
path : str
path to the json object to extract
@@ -3437,7 +3440,7 @@ def json_tuple(col, *fields):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
string column in json format
fields : str
fields to extract
@@ -3464,7 +3467,7 @@ def from_json(col, schema, options={}):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
string column in json format
schema : :class:`DataType` or str
a StructType or ArrayType of StructType to use when parsing the json column.
@@ -3519,7 +3522,7 @@ def to_json(col, options={}):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column containing a struct, an array or a map.
options : dict, optional
options to control converting. accepts the same options as the JSON datasource.
@@ -3565,7 +3568,7 @@ def schema_of_json(json, options={}):
Parameters
----------
- json : :class:`Column` or str
+ json : :class:`~pyspark.sql.Column` or str
a JSON string or a foldable string column containing a JSON string.
options : dict, optional
options to control parsing. accepts the same options as the JSON datasource
@@ -3602,7 +3605,7 @@ def schema_of_csv(csv, options={}):
Parameters
----------
- csv : :class:`Column` or str
+ csv : :class:`~pyspark.sql.Column` or str
a CSV string or a foldable string column containing a CSV string.
options : dict, optional
options to control parsing. accepts the same options as the CSV datasource
@@ -3636,7 +3639,7 @@ def to_csv(col, options={}):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column containing a struct.
options: dict, optional
options to control converting. accepts the same options as the CSV datasource.
@@ -3663,7 +3666,7 @@ def size(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3684,7 +3687,7 @@ def array_min(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3705,7 +3708,7 @@ def array_max(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3729,7 +3732,7 @@ def sort_array(col, asc=True):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
asc : bool, optional
@@ -3754,7 +3757,7 @@ def array_sort(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3775,7 +3778,7 @@ def shuffle(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Notes
@@ -3800,7 +3803,7 @@ def reverse(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3826,7 +3829,7 @@ def flatten(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3847,7 +3850,7 @@ def map_keys(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3873,7 +3876,7 @@ def map_values(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3899,7 +3902,7 @@ def map_entries(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3925,7 +3928,7 @@ def map_from_entries(col):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
Examples
@@ -3971,7 +3974,7 @@ def arrays_zip(*cols):
Parameters
----------
- cols : :class:`Column` or str
+ cols : :class:`~pyspark.sql.Column` or str
columns of arrays to be merged.
Examples
@@ -3992,8 +3995,8 @@ def map_concat(*cols):
Parameters
----------
- cols : :class:`Column` or str
- column names or :class:`Column`\\s
+ cols : :class:`~pyspark.sql.Column` or str
+ column names or :class:`~pyspark.sql.Column`\\s
Examples
--------
@@ -4047,9 +4050,9 @@ def from_csv(col, schema, options={}):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
string column in CSV format
- schema :class:`Column` or str
+ schema :class:`~pyspark.sql.Column` or str
a string with schema in DDL format to use when parsing the CSV column.
options : dict, optional
options to control parsing. accepts the same options as the CSV datasource
@@ -4194,7 +4197,7 @@ def transform(col, f):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
f : function
a function that is applied to each element of the input array.
@@ -4204,14 +4207,14 @@ def transform(col, f):
- Binary ``(x: Column, i: Column) -> Column...``, where the second argument is
a 0-based index of the element.
- and can use methods of :class:`pyspark.sql.Column`, functions defined in
+ and can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
@@ -4243,15 +4246,15 @@ def exists(col, f):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
f : function
``(x: Column) -> Column: ...`` returning the Boolean expression.
- Can use methods of :class:`pyspark.sql.Column`, functions defined in
+ Can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- :return: a :class:`pyspark.sql.Column`
+ :return: a :class:`~pyspark.sql.Column`
Examples
--------
@@ -4275,18 +4278,18 @@ def forall(col, f):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
f : function
``(x: Column) -> Column: ...`` returning the Boolean expression.
- Can use methods of :class:`pyspark.sql.Column`, functions defined in
+ Can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
@@ -4314,7 +4317,7 @@ def filter(col, f):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
f : function
A function that returns the Boolean expression.
@@ -4324,14 +4327,14 @@ def filter(col, f):
- Binary ``(x: Column, i: Column) -> Column...``, where the second argument is
a 0-based index of the element.
- and can use methods of :class:`pyspark.sql.Column`, functions defined in
+ and can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
@@ -4359,7 +4362,7 @@ def aggregate(col, initialValue, merge, finish=None):
and reduces this to a single state. The final state is converted into the final result
by applying a finish function.
- Both functions can use methods of :class:`pyspark.sql.Column`, functions defined in
+ Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
@@ -4368,9 +4371,9 @@ def aggregate(col, initialValue, merge, finish=None):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
- initialValue : :class:`Column` or str
+ initialValue : :class:`~pyspark.sql.Column` or str
initial value. Name of column or expression
merge : function
a binary function ``(acc: Column, x: Column) -> Column...`` returning expression
@@ -4381,7 +4384,7 @@ def aggregate(col, initialValue, merge, finish=None):
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
@@ -4436,20 +4439,20 @@ def zip_with(left, right, f):
Parameters
----------
- left : :class:`Column` or str
+ left : :class:`~pyspark.sql.Column` or str
name of the first column or expression
- right : :class:`Column` or str
+ right : :class:`~pyspark.sql.Column` or str
name of the second column or expression
f : function
a binary function ``(x1: Column, x2: Column) -> Column...``
- Can use methods of :class:`pyspark.sql.Column`, functions defined in
+ Can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
@@ -4481,18 +4484,18 @@ def transform_keys(col, f):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
f : function
a binary function ``(k: Column, v: Column) -> Column...``
- Can use methods of :class:`pyspark.sql.Column`, functions defined in
+ Can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
@@ -4518,18 +4521,18 @@ def transform_values(col, f):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
f : function
a binary function ``(k: Column, v: Column) -> Column...``
- Can use methods of :class:`pyspark.sql.Column`, functions defined in
+ Can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
@@ -4554,18 +4557,18 @@ def map_filter(col, f):
Parameters
----------
- col : :class:`Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column or expression
f : function
a binary function ``(k: Column, v: Column) -> Column...``
- Can use methods of :class:`pyspark.sql.Column`, functions defined in
+ Can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
@@ -4590,20 +4593,20 @@ def map_zip_with(col1, col2, f):
Parameters
----------
- col1 : :class:`Column` or str
+ col1 : :class:`~pyspark.sql.Column` or str
name of the first column or expression
- col2 : :class:`Column` or str
+ col2 : :class:`~pyspark.sql.Column` or str
name of the second column or expression
f : function
a ternary function ``(k: Column, v1: Column, v2: Column) -> Column...``
- Can use methods of :class:`pyspark.sql.Column`, functions defined in
+ Can use methods of :class:`~pyspark.sql.Column`, functions defined in
:py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
Python ``UserDefinedFunctions`` are not supported
(`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
Returns
-------
- :class:`pyspark.sql.Column`
+ :class:`~pyspark.sql.Column`
Examples
--------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org