You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/11/29 01:02:05 UTC
[spark] branch master updated: [SPARK-41304][CONNECT][PYTHON][DOCS] Add missing docs for DataFrame API
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c78676e82e2 [SPARK-41304][CONNECT][PYTHON][DOCS] Add missing docs for DataFrame API
c78676e82e2 is described below
commit c78676e82e2ec041270519a2d53555591da078ab
Author: Rui Wang <ru...@databricks.com>
AuthorDate: Tue Nov 29 10:01:52 2022 +0900
[SPARK-41304][CONNECT][PYTHON][DOCS] Add missing docs for DataFrame API
### What changes were proposed in this pull request?
Add missing docs for DataFrame API.
### Why are the changes needed?
Improve documentation
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
N/A
Closes #38824 from amaliujia/improve_pyspark_doc.
Authored-by: Rui Wang <ru...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/connect/dataframe.py | 142 +++++++++++++++++++++++++++++++-
1 file changed, 141 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index d77a3717248..068c6322698 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -309,6 +309,21 @@ class DataFrame(object):
)
def drop(self, *cols: "ColumnOrName") -> "DataFrame":
+ """Returns a new :class:`DataFrame` without specified columns.
+ This is a no-op if schema doesn't contain the given column name(s).
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ cols: str or :class:`Column`
+ a name of the column, or the :class:`Column` to drop
+
+ Returns
+ -------
+ :class:`DataFrame`
+ DataFrame without given columns.
+ """
_cols = list(cols)
if any(not isinstance(c, (str, Column)) for c in _cols):
raise TypeError(
@@ -326,6 +341,23 @@ class DataFrame(object):
)
def filter(self, condition: Expression) -> "DataFrame":
+ """Filters rows using the given condition.
+
+ :func:`where` is an alias for :func:`filter`.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ condition : :class:`Column` or str
+ a :class:`Column` of :class:`types.BooleanType`
+ or a string of SQL expression.
+
+ Returns
+ -------
+ :class:`DataFrame`
+ Filtered DataFrame.
+ """
return DataFrame.withPlan(
plan.Filter(child=self._plan, filter=condition), session=self._session
)
@@ -409,9 +441,39 @@ class DataFrame(object):
)
def limit(self, n: int) -> "DataFrame":
+ """Limits the result count to the number specified.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ num : int
+ Number of records to return. Will return this number of records
+ or whatever number is available.
+
+ Returns
+ -------
+ :class:`DataFrame`
+ Subset of the records
+ """
return DataFrame.withPlan(plan.Limit(child=self._plan, limit=n), session=self._session)
def offset(self, n: int) -> "DataFrame":
+ """Returns a new :class: `DataFrame` by skipping the first `n` rows.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ num : int
+ Number of records to return. Will return this number of records
+ or all records if the DataFrame contains less than this number of records.
+
+ Returns
+ -------
+ :class:`DataFrame`
+ Subset of the records
+ """
return DataFrame.withPlan(plan.Offset(child=self._plan, offset=n), session=self._session)
def tail(self, num: int) -> List[Row]:
@@ -427,7 +489,7 @@ class DataFrame(object):
----------
num : int
Number of records to return. Will return this number of records
- or whataver number is available.
+ or all records if the DataFrame contains less than this number of records.
Returns
-------
@@ -457,6 +519,31 @@ class DataFrame(object):
withReplacement: bool = False,
seed: Optional[int] = None,
) -> "DataFrame":
+ """Returns a sampled subset of this :class:`DataFrame`.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ withReplacement : bool, optional
+ Sample with replacement or not (default ``False``).
+ fraction : float
+ Fraction of rows to generate, range [0.0, 1.0].
+ seed : int, optional
+ Seed for sampling (default a random seed).
+
+ Returns
+ -------
+ :class:`DataFrame`
+ Sampled rows from given DataFrame.
+
+ Notes
+ -----
+ This is not guaranteed to provide exactly the fraction specified of the total
+ count of the given :class:`DataFrame`.
+
+ `fraction` is required and, `withReplacement` and `seed` are optional.
+ """
if not isinstance(fraction, float):
raise TypeError(f"'fraction' must be float, but got {type(fraction).__name__}")
if not isinstance(withReplacement, bool):
@@ -524,9 +611,62 @@ class DataFrame(object):
print(self._show_string(n, truncate, vertical))
def union(self, other: "DataFrame") -> "DataFrame":
+ """Return a new :class:`DataFrame` containing union of rows in this and another
+ :class:`DataFrame`.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ other : :class:`DataFrame`
+ Another :class:`DataFrame` that needs to be unioned
+
+ Returns
+ -------
+ :class:`DataFrame`
+
+ See Also
+ --------
+ DataFrame.unionAll
+
+ Notes
+ -----
+ This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union
+ (that does deduplication of elements), use this function followed by :func:`distinct`.
+
+ Also as standard in SQL, this function resolves columns by position (not by name).
+ """
return self.unionAll(other)
def unionAll(self, other: "DataFrame") -> "DataFrame":
+ """Return a new :class:`DataFrame` containing union of rows in this and another
+ :class:`DataFrame`.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ other : :class:`DataFrame`
+ Another :class:`DataFrame` that needs to be combined
+
+ Returns
+ -------
+ :class:`DataFrame`
+ Combined DataFrame
+
+ Notes
+ -----
+ This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union
+ (that does deduplication of elements), use this function followed by :func:`distinct`.
+
+ Also as standard in SQL, this function resolves columns by position (not by name).
+
+ :func:`unionAll` is an alias to :func:`union`
+
+ See Also
+ --------
+ DataFrame.union
+ """
if other._plan is None:
raise ValueError("Argument to Union does not contain a valid plan.")
return DataFrame.withPlan(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org