You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2022/11/28 07:48:54 UTC
[spark] branch master updated: [SPARK-41291][CONNECT][PYTHON] DataFrame.explain` should print and return None
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1a8f2952225 [SPARK-41291][CONNECT][PYTHON] DataFrame.explain` should print and return None
1a8f2952225 is described below
commit 1a8f295222503f869ea9bb919f97202103ddd3c4
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Mon Nov 28 15:48:32 2022 +0800
[SPARK-41291][CONNECT][PYTHON] DataFrame.explain` should print and return None
### What changes were proposed in this pull request?
`DataFrame.explain` should print and return None
### Why are the changes needed?
to match the behavior in PySpark
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
updated tests
Closes #38816 from zhengruifeng/connect_explain_print.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
python/pyspark/sql/connect/dataframe.py | 47 ++++++++++++----------
.../sql/tests/connect/test_connect_basic.py | 10 +++--
2 files changed, 32 insertions(+), 25 deletions(-)
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index 862bb566ebe..d77a3717248 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -975,29 +975,9 @@ class DataFrame(object):
), "Func returned an instance of type [%s], " "should have been DataFrame." % type(result)
return result
- def explain(
+ def _explain_string(
self, extended: Optional[Union[bool, str]] = None, mode: Optional[str] = None
) -> str:
- """Retruns plans in string for debugging purpose.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- extended : bool, optional
- default ``False``. If ``False``, returns only the physical plan.
- When this is a string without specifying the ``mode``, it works as the mode is
- specified.
- mode : str, optional
- specifies the expected output format of plans.
-
- * ``simple``: Print only a physical plan.
- * ``extended``: Print both logical and physical plans.
- * ``codegen``: Print a physical plan and generated codes if they are available.
- * ``cost``: Print a logical plan and statistics if they are available.
- * ``formatted``: Split explain output into two sections: a physical plan outline \
- and node details.
- """
if extended is not None and mode is not None:
raise ValueError("extended and mode should not be set together.")
@@ -1042,6 +1022,31 @@ class DataFrame(object):
else:
return ""
+ def explain(
+ self, extended: Optional[Union[bool, str]] = None, mode: Optional[str] = None
+ ) -> None:
+ """Retruns plans in string for debugging purpose.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ extended : bool, optional
+ default ``False``. If ``False``, returns only the physical plan.
+ When this is a string without specifying the ``mode``, it works as the mode is
+ specified.
+ mode : str, optional
+ specifies the expected output format of plans.
+
+ * ``simple``: Print only a physical plan.
+ * ``extended``: Print both logical and physical plans.
+ * ``codegen``: Print a physical plan and generated codes if they are available.
+ * ``cost``: Print a logical plan and statistics if they are available.
+ * ``formatted``: Split explain output into two sections: a physical plan outline \
+ and node details.
+ """
+ print(self._explain_string(extended=extended, mode=mode))
+
def createGlobalTempView(self, name: str) -> None:
"""Creates a global temporary view with this :class:`DataFrame`.
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py
index 187788fd6ab..0b07a8328a1 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -133,7 +133,7 @@ class SparkConnectTests(SparkConnectSQLTestCase):
def test_simple_explain_string(self):
df = self.connect.read.table(self.tbl_name).limit(10)
- result = df.explain()
+ result = df._explain_string()
self.assertGreater(len(result), 0)
def test_schema(self):
@@ -330,7 +330,9 @@ class SparkConnectTests(SparkConnectSQLTestCase):
def test_subquery_alias(self) -> None:
# SPARK-40938: test subquery alias.
plan_text = (
- self.connect.read.table(self.tbl_name).alias("special_alias").explain(extended=True)
+ self.connect.read.table(self.tbl_name)
+ .alias("special_alias")
+ ._explain_string(extended=True)
)
self.assertTrue("special_alias" in plan_text)
@@ -511,14 +513,14 @@ class SparkConnectTests(SparkConnectSQLTestCase):
def test_explain_string(self):
# SPARK-41122: test explain API.
- plan_str = self.connect.sql("SELECT 1").explain(extended=True)
+ plan_str = self.connect.sql("SELECT 1")._explain_string(extended=True)
self.assertTrue("Parsed Logical Plan" in plan_str)
self.assertTrue("Analyzed Logical Plan" in plan_str)
self.assertTrue("Optimized Logical Plan" in plan_str)
self.assertTrue("Physical Plan" in plan_str)
with self.assertRaises(ValueError) as context:
- self.connect.sql("SELECT 1").explain(mode="unknown")
+ self.connect.sql("SELECT 1")._explain_string(mode="unknown")
self.assertTrue("unknown" in str(context.exception))
def test_simple_datasource_read(self) -> None:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org