You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2022/11/28 07:48:54 UTC

[spark] branch master updated: [SPARK-41291][CONNECT][PYTHON] DataFrame.explain` should print and return None

This is an automated email from the ASF dual-hosted git repository.

ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 1a8f2952225 [SPARK-41291][CONNECT][PYTHON] DataFrame.explain` should print and return None
1a8f2952225 is described below

commit 1a8f295222503f869ea9bb919f97202103ddd3c4
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Mon Nov 28 15:48:32 2022 +0800

    [SPARK-41291][CONNECT][PYTHON] DataFrame.explain` should print and return None
    
    ### What changes were proposed in this pull request?
    `DataFrame.explain` should print and return None
    
    ### Why are the changes needed?
    to match the behavior in PySpark
    
    ### Does this PR introduce _any_ user-facing change?
    no
    
    ### How was this patch tested?
    updated tests
    
    Closes #38816 from zhengruifeng/connect_explain_print.
    
    Authored-by: Ruifeng Zheng <ru...@apache.org>
    Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
 python/pyspark/sql/connect/dataframe.py            | 47 ++++++++++++----------
 .../sql/tests/connect/test_connect_basic.py        | 10 +++--
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index 862bb566ebe..d77a3717248 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -975,29 +975,9 @@ class DataFrame(object):
         ), "Func returned an instance of type [%s], " "should have been DataFrame." % type(result)
         return result
 
-    def explain(
+    def _explain_string(
         self, extended: Optional[Union[bool, str]] = None, mode: Optional[str] = None
     ) -> str:
-        """Retruns plans in string for debugging purpose.
-
-        .. versionadded:: 3.4.0
-
-        Parameters
-        ----------
-        extended : bool, optional
-            default ``False``. If ``False``, returns only the physical plan.
-            When this is a string without specifying the ``mode``, it works as the mode is
-            specified.
-        mode : str, optional
-            specifies the expected output format of plans.
-
-            * ``simple``: Print only a physical plan.
-            * ``extended``: Print both logical and physical plans.
-            * ``codegen``: Print a physical plan and generated codes if they are available.
-            * ``cost``: Print a logical plan and statistics if they are available.
-            * ``formatted``: Split explain output into two sections: a physical plan outline \
-                and node details.
-        """
         if extended is not None and mode is not None:
             raise ValueError("extended and mode should not be set together.")
 
@@ -1042,6 +1022,31 @@ class DataFrame(object):
         else:
             return ""
 
+    def explain(
+        self, extended: Optional[Union[bool, str]] = None, mode: Optional[str] = None
+    ) -> None:
+        """Retruns plans in string for debugging purpose.
+
+        .. versionadded:: 3.4.0
+
+        Parameters
+        ----------
+        extended : bool, optional
+            default ``False``. If ``False``, returns only the physical plan.
+            When this is a string without specifying the ``mode``, it works as the mode is
+            specified.
+        mode : str, optional
+            specifies the expected output format of plans.
+
+            * ``simple``: Print only a physical plan.
+            * ``extended``: Print both logical and physical plans.
+            * ``codegen``: Print a physical plan and generated codes if they are available.
+            * ``cost``: Print a logical plan and statistics if they are available.
+            * ``formatted``: Split explain output into two sections: a physical plan outline \
+                and node details.
+        """
+        print(self._explain_string(extended=extended, mode=mode))
+
     def createGlobalTempView(self, name: str) -> None:
         """Creates a global temporary view with this :class:`DataFrame`.
 
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py
index 187788fd6ab..0b07a8328a1 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -133,7 +133,7 @@ class SparkConnectTests(SparkConnectSQLTestCase):
 
     def test_simple_explain_string(self):
         df = self.connect.read.table(self.tbl_name).limit(10)
-        result = df.explain()
+        result = df._explain_string()
         self.assertGreater(len(result), 0)
 
     def test_schema(self):
@@ -330,7 +330,9 @@ class SparkConnectTests(SparkConnectSQLTestCase):
     def test_subquery_alias(self) -> None:
         # SPARK-40938: test subquery alias.
         plan_text = (
-            self.connect.read.table(self.tbl_name).alias("special_alias").explain(extended=True)
+            self.connect.read.table(self.tbl_name)
+            .alias("special_alias")
+            ._explain_string(extended=True)
         )
         self.assertTrue("special_alias" in plan_text)
 
@@ -511,14 +513,14 @@ class SparkConnectTests(SparkConnectSQLTestCase):
 
     def test_explain_string(self):
         # SPARK-41122: test explain API.
-        plan_str = self.connect.sql("SELECT 1").explain(extended=True)
+        plan_str = self.connect.sql("SELECT 1")._explain_string(extended=True)
         self.assertTrue("Parsed Logical Plan" in plan_str)
         self.assertTrue("Analyzed Logical Plan" in plan_str)
         self.assertTrue("Optimized Logical Plan" in plan_str)
         self.assertTrue("Physical Plan" in plan_str)
 
         with self.assertRaises(ValueError) as context:
-            self.connect.sql("SELECT 1").explain(mode="unknown")
+            self.connect.sql("SELECT 1")._explain_string(mode="unknown")
         self.assertTrue("unknown" in str(context.exception))
 
     def test_simple_datasource_read(self) -> None:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org