You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2023/05/23 08:43:03 UTC
[spark] branch master updated: [SPARK-43625][DOCS] Document the difference between `Drop(column)` and `Drop(columnName)`
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8f26ba03b9a [SPARK-43625][DOCS] Document the difference between `Drop(column)` and `Drop(columnName)`
8f26ba03b9a is described below
commit 8f26ba03b9ae562b614eae95b7c793a88b39f650
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Tue May 23 01:42:28 2023 -0700
[SPARK-43625][DOCS] Document the difference between `Drop(column)` and `Drop(columnName)`
### What changes were proposed in this pull request?
Document the difference between `Drop(column)` and `Drop(columnName)`
### Why are the changes needed?
to better illustrate this difference
### Does this PR introduce _any_ user-facing change?
yes, new doc and example
### How was this patch tested?
CI and added doctest
Closes #41273 from zhengruifeng/doc_drop_api.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
python/pyspark/sql/dataframe.py | 72 ++++++++++++++++++++++
.../main/scala/org/apache/spark/sql/Dataset.scala | 64 +++++++++++++++++++
2 files changed, 136 insertions(+)
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 4a20566771b..d98f025c50c 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -5116,9 +5116,17 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
:class:`DataFrame`
DataFrame without given columns.
+ Notes
+ -----
+ When an input is a column name, it is treated literally without further interpretation.
+ Otherwise, will try to match the equivalent expression.
+ So that dropping column by its name `drop(colName)` has different semantic with directly
+ dropping the column `drop(col(colName))`.
+
Examples
--------
>>> from pyspark.sql import Row
+ >>> from pyspark.sql.functions import col, lit
>>> df = spark.createDataFrame(
... [(14, "Tom"), (23, "Alice"), (16, "Bob")], ["age", "name"])
>>> df2 = spark.createDataFrame([Row(height=80, name="Tom"), Row(height=85, name="Bob")])
@@ -5149,6 +5157,70 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
| 14| 80|
| 16| 85|
+---+------+
+
+ >>> df3 = df.join(df2)
+ >>> df3.show()
+ +---+-----+------+----+
+ |age| name|height|name|
+ +---+-----+------+----+
+ | 14| Tom| 80| Tom|
+ | 14| Tom| 85| Bob|
+ | 23|Alice| 80| Tom|
+ | 23|Alice| 85| Bob|
+ | 16| Bob| 80| Tom|
+ | 16| Bob| 85| Bob|
+ +---+-----+------+----+
+
+ Drop two column by the same name.
+
+ >>> df3.drop("name").show()
+ +---+------+
+ |age|height|
+ +---+------+
+ | 14| 80|
+ | 14| 85|
+ | 23| 80|
+ | 23| 85|
+ | 16| 80|
+ | 16| 85|
+ +---+------+
+
+ Can not drop col('name') due to ambiguous reference.
+
+ >>> df3.drop(col("name")).show()
+ Traceback (most recent call last):
+ ...
+ pyspark.errors.exceptions.captured.AnalysisException: [AMBIGUOUS_REFERENCE] Reference...
+
+ >>> df4 = df.withColumn("a.b.c", lit(1))
+ >>> df4.show()
+ +---+-----+-----+
+ |age| name|a.b.c|
+ +---+-----+-----+
+ | 14| Tom| 1|
+ | 23|Alice| 1|
+ | 16| Bob| 1|
+ +---+-----+-----+
+
+ >>> df4.drop("a.b.c").show()
+ +---+-----+
+ |age| name|
+ +---+-----+
+ | 14| Tom|
+ | 23|Alice|
+ | 16| Bob|
+ +---+-----+
+
+ Can not find a column matching the expression "a.b.c".
+
+ >>> df4.drop(col("a.b.c")).show()
+ +---+-----+-----+
+ |age| name|a.b.c|
+ +---+-----+-----+
+ | 14| Tom| 1|
+ | 23|Alice| 1|
+ | 16| Bob| 1|
+ +---+-----+-----+
"""
column_names: List[str] = []
java_columns: List[JavaObject] = []
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 7137e24ed96..9f204658175 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -2932,6 +2932,67 @@ class Dataset[T] private[sql](
* This method can only be used to drop top level columns. the colName string is treated
* literally without further interpretation.
*
+ * Note: `drop(colName)` has different semantic with `drop(col(colName))`, for example:
+ * 1, multi column have the same colName:
+ * {{{
+ * val df1 = spark.range(0, 2).withColumn("key1", lit(1))
+ * val df2 = spark.range(0, 2).withColumn("key2", lit(2))
+ * val df3 = df1.join(df2)
+ *
+ * df3.show
+ * // +---+----+---+----+
+ * // | id|key1| id|key2|
+ * // +---+----+---+----+
+ * // | 0| 1| 0| 2|
+ * // | 0| 1| 1| 2|
+ * // | 1| 1| 0| 2|
+ * // | 1| 1| 1| 2|
+ * // +---+----+---+----+
+ *
+ * df3.drop("id").show()
+ * // output: the two 'id' columns are both dropped.
+ * // |key1|key2|
+ * // +----+----+
+ * // | 1| 2|
+ * // | 1| 2|
+ * // | 1| 2|
+ * // | 1| 2|
+ * // +----+----+
+ *
+ * df3.drop(col("id")).show()
+ * // ...AnalysisException: [AMBIGUOUS_REFERENCE] Reference `id` is ambiguous...
+ * }}}
+ *
+ * 2, colName contains special characters, like dot.
+ * {{{
+ * val df = spark.range(0, 2).withColumn("a.b.c", lit(1))
+ *
+ * df.show()
+ * // +---+-----+
+ * // | id|a.b.c|
+ * // +---+-----+
+ * // | 0| 1|
+ * // | 1| 1|
+ * // +---+-----+
+ *
+ * df.drop("a.b.c").show()
+ * // +---+
+ * // | id|
+ * // +---+
+ * // | 0|
+ * // | 1|
+ * // +---+
+ *
+ * df.drop(col("a.b.c")).show()
+ * // no column match the expression 'a.b.c'
+ * // +---+-----+
+ * // | id|a.b.c|
+ * // +---+-----+
+ * // | 0| 1|
+ * // | 1| 1|
+ * // +---+-----+
+ * }}}
+ *
* @group untypedrel
* @since 2.0.0
*/
@@ -2971,6 +3032,9 @@ class Dataset[T] private[sql](
* This is a no-op if the Dataset doesn't have a column
* with an equivalent expression.
*
+ * Note: `drop(col(colName))` has different semantic with `drop(colName)`,
+ * please refer to `Dataset#drop(colName: String)`.
+ *
* @group untypedrel
* @since 2.0.0
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org