You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2021/07/20 20:45:33 UTC

[arrow-datafusion] branch master updated: Show the result of all optimizer passes in EXPLAIN VERBOSE (#759)

This is an automated email from the ASF dual-hosted git repository.

jorgecarleitao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/master by this push:
     new 30693df  Show the result of all optimizer passes in EXPLAIN VERBOSE (#759)
30693df is described below

commit 30693df8961dca300306dfd0c8fca130375b50b3
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Tue Jul 20 16:43:51 2021 -0400

    Show the result of all optimizer passes in EXPLAIN VERBOSE (#759)
---
 datafusion/src/execution/context.rs              | 106 ++++++++++++++++++---
 datafusion/src/logical_plan/builder.rs           |  15 ++-
 datafusion/src/logical_plan/mod.rs               |   2 +-
 datafusion/src/logical_plan/plan.rs              |  20 ++++
 datafusion/src/optimizer/limit_push_down.rs      |  20 ----
 datafusion/src/optimizer/projection_push_down.rs |  22 +----
 datafusion/src/optimizer/simplify_expressions.rs |  23 +----
 datafusion/src/optimizer/utils.rs                | 111 +---------------------
 datafusion/src/physical_plan/display.rs          |  11 +++
 datafusion/src/physical_plan/explain.rs          |  24 ++++-
 datafusion/src/physical_plan/planner.rs          | 115 +++++++++++++++--------
 datafusion/src/sql/planner.rs                    |   9 +-
 datafusion/tests/sql.rs                          |   7 ++
 datafusion/tests/user_defined_plan.rs            |   6 +-
 14 files changed, 252 insertions(+), 239 deletions(-)

diff --git a/datafusion/src/execution/context.rs b/datafusion/src/execution/context.rs
index bd939ce..0cf8b3b 100644
--- a/datafusion/src/execution/context.rs
+++ b/datafusion/src/execution/context.rs
@@ -21,6 +21,7 @@ use crate::{
         catalog::{CatalogList, MemoryCatalogList},
         information_schema::CatalogWithInformationSchema,
     },
+    logical_plan::{PlanType, ToStringifiedPlan},
     optimizer::{
         aggregate_statistics::AggregateStatistics, eliminate_limit::EliminateLimit,
         hash_build_probe_order::HashBuildProbeOrder,
@@ -446,19 +447,31 @@ impl ExecutionContext {
 
     /// Optimizes the logical plan by applying optimizer rules.
     pub fn optimize(&self, plan: &LogicalPlan) -> Result<LogicalPlan> {
-        let state = &mut self.state.lock().unwrap();
-        let execution_props = &mut state.execution_props.clone();
-        let optimizers = &state.config.optimizers;
-
-        let execution_props = execution_props.start_execution();
-
-        let mut new_plan = plan.clone();
-        debug!("Logical plan:\n {:?}", plan);
-        for optimizer in optimizers {
-            new_plan = optimizer.optimize(&new_plan, execution_props)?;
+        if let LogicalPlan::Explain {
+            verbose,
+            plan,
+            stringified_plans,
+            schema,
+        } = plan
+        {
+            let mut stringified_plans = stringified_plans.clone();
+
+            // optimize the child plan, capturing the output of each optimizer
+            let plan = self.optimize_internal(plan, |optimized_plan, optimizer| {
+                let optimizer_name = optimizer.name().to_string();
+                let plan_type = PlanType::OptimizedLogicalPlan { optimizer_name };
+                stringified_plans.push(optimized_plan.to_stringified(plan_type));
+            })?;
+
+            Ok(LogicalPlan::Explain {
+                verbose: *verbose,
+                plan: Arc::new(plan),
+                stringified_plans,
+                schema: schema.clone(),
+            })
+        } else {
+            self.optimize_internal(plan, |_, _| {})
         }
-        debug!("Optimized logical plan:\n {:?}", new_plan);
-        Ok(new_plan)
     }
 
     /// Creates a physical plan from a logical plan.
@@ -556,6 +569,32 @@ impl ExecutionContext {
             ))),
         }
     }
+
+    /// Optimizes the logical plan by applying optimizer rules, and
+    /// invoking observer function after each call
+    fn optimize_internal<F>(
+        &self,
+        plan: &LogicalPlan,
+        mut observer: F,
+    ) -> Result<LogicalPlan>
+    where
+        F: FnMut(&LogicalPlan, &dyn OptimizerRule),
+    {
+        let state = &mut self.state.lock().unwrap();
+        let execution_props = &mut state.execution_props.clone();
+        let optimizers = &state.config.optimizers;
+
+        let execution_props = execution_props.start_execution();
+
+        let mut new_plan = plan.clone();
+        debug!("Logical plan:\n {:?}", plan);
+        for optimizer in optimizers {
+            new_plan = optimizer.optimize(&new_plan, execution_props)?;
+            observer(&new_plan, optimizer.as_ref());
+        }
+        debug!("Optimized logical plan:\n {:?}", new_plan);
+        Ok(new_plan)
+    }
 }
 
 impl From<Arc<Mutex<ExecutionContextState>>> for ExecutionContext {
@@ -941,6 +980,49 @@ mod tests {
     use tempfile::TempDir;
     use test::*;
 
+    #[test]
+    fn optimize_explain() {
+        let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]);
+
+        let plan = LogicalPlanBuilder::scan_empty(Some("employee"), &schema, None)
+            .unwrap()
+            .explain(true)
+            .unwrap()
+            .build()
+            .unwrap();
+
+        if let LogicalPlan::Explain {
+            stringified_plans, ..
+        } = &plan
+        {
+            assert_eq!(stringified_plans.len(), 1);
+        } else {
+            panic!("plan was not an explain: {:?}", plan);
+        }
+
+        // now optimize the plan and expect to see more plans
+        let optimized_plan = ExecutionContext::new().optimize(&plan).unwrap();
+        if let LogicalPlan::Explain {
+            stringified_plans, ..
+        } = &optimized_plan
+        {
+            // should have more than one plan
+            assert!(
+                stringified_plans.len() > 1,
+                "plans: {:#?}",
+                stringified_plans
+            );
+            // should have at least one optimized plan
+            let opt = stringified_plans
+                .iter()
+                .any(|p| matches!(p.plan_type, PlanType::OptimizedLogicalPlan { .. }));
+
+            assert!(opt, "plans: {:#?}", stringified_plans);
+        } else {
+            panic!("plan was not an explain: {:?}", plan);
+        }
+    }
+
     #[tokio::test]
     async fn parallel_projection() -> Result<()> {
         let partition_count = 4;
diff --git a/datafusion/src/logical_plan/builder.rs b/datafusion/src/logical_plan/builder.rs
index 0335e29..60e0ed3 100644
--- a/datafusion/src/logical_plan/builder.rs
+++ b/datafusion/src/logical_plan/builder.rs
@@ -27,18 +27,15 @@ use arrow::{
     record_batch::RecordBatch,
 };
 
-use crate::datasource::TableProvider;
 use crate::error::{DataFusionError, Result};
+use crate::{datasource::TableProvider, logical_plan::plan::ToStringifiedPlan};
 use crate::{
     datasource::{empty::EmptyTable, parquet::ParquetTable, CsvFile, MemTable},
     prelude::CsvReadOptions,
 };
 
 use super::dfschema::ToDFSchema;
-use super::{
-    exprlist_to_fields, Expr, JoinConstraint, JoinType, LogicalPlan, PlanType,
-    StringifiedPlan,
-};
+use super::{exprlist_to_fields, Expr, JoinConstraint, JoinType, LogicalPlan, PlanType};
 use crate::logical_plan::{
     columnize_expr, normalize_col, normalize_cols, Column, DFField, DFSchema,
     DFSchemaRef, Partitioning,
@@ -398,10 +395,8 @@ impl LogicalPlanBuilder {
 
     /// Create an expression to represent the explanation of the plan
     pub fn explain(&self, verbose: bool) -> Result<Self> {
-        let stringified_plans = vec![StringifiedPlan::new(
-            PlanType::InitialLogicalPlan,
-            format!("{:#?}", self.plan.clone()),
-        )];
+        let stringified_plans =
+            vec![self.plan.to_stringified(PlanType::InitialLogicalPlan)];
 
         let schema = LogicalPlan::explain_schema();
 
@@ -553,6 +548,8 @@ pub(crate) fn expand_wildcard(
 mod tests {
     use arrow::datatypes::{DataType, Field};
 
+    use crate::logical_plan::StringifiedPlan;
+
     use super::super::{col, lit, sum};
     use super::*;
 
diff --git a/datafusion/src/logical_plan/mod.rs b/datafusion/src/logical_plan/mod.rs
index f381e31..a021d06 100644
--- a/datafusion/src/logical_plan/mod.rs
+++ b/datafusion/src/logical_plan/mod.rs
@@ -50,6 +50,6 @@ pub use extension::UserDefinedLogicalNode;
 pub use operators::Operator;
 pub use plan::{
     JoinConstraint, JoinType, LogicalPlan, Partitioning, PlanType, PlanVisitor,
-    StringifiedPlan,
 };
+pub(crate) use plan::{StringifiedPlan, ToStringifiedPlan};
 pub use registry::FunctionRegistry;
diff --git a/datafusion/src/logical_plan/plan.rs b/datafusion/src/logical_plan/plan.rs
index 9a4daae..28405fb 100644
--- a/datafusion/src/logical_plan/plan.rs
+++ b/datafusion/src/logical_plan/plan.rs
@@ -820,6 +820,11 @@ pub enum PlanType {
     FinalLogicalPlan,
     /// The initial physical plan, prepared for execution
     InitialPhysicalPlan,
+    /// The ExecutionPlan which results from applying an optimizer pass
+    OptimizedPhysicalPlan {
+        /// The name of the optimizer which produced this plan
+        optimizer_name: String,
+    },
     /// The final, fully optimized physical which would be executed
     FinalPhysicalPlan,
 }
@@ -833,6 +838,9 @@ impl fmt::Display for PlanType {
             }
             PlanType::FinalLogicalPlan => write!(f, "logical_plan"),
             PlanType::InitialPhysicalPlan => write!(f, "initial_physical_plan"),
+            PlanType::OptimizedPhysicalPlan { optimizer_name } => {
+                write!(f, "physical_plan after {}", optimizer_name)
+            }
             PlanType::FinalPhysicalPlan => write!(f, "physical_plan"),
         }
     }
@@ -868,6 +876,18 @@ impl StringifiedPlan {
     }
 }
 
+/// Trait for something that can be formatted as a stringified plan
+pub trait ToStringifiedPlan {
+    /// Create a stringified plan with the specified type
+    fn to_stringified(&self, plan_type: PlanType) -> StringifiedPlan;
+}
+
+impl ToStringifiedPlan for LogicalPlan {
+    fn to_stringified(&self, plan_type: PlanType) -> StringifiedPlan {
+        StringifiedPlan::new(plan_type, self.display_indent().to_string())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::super::{col, lit, LogicalPlanBuilder};
diff --git a/datafusion/src/optimizer/limit_push_down.rs b/datafusion/src/optimizer/limit_push_down.rs
index 37c95a4..46738c5 100644
--- a/datafusion/src/optimizer/limit_push_down.rs
+++ b/datafusion/src/optimizer/limit_push_down.rs
@@ -23,7 +23,6 @@ use crate::execution::context::ExecutionProps;
 use crate::logical_plan::LogicalPlan;
 use crate::optimizer::optimizer::OptimizerRule;
 use std::sync::Arc;
-use utils::optimize_explain;
 
 /// Optimization rule that tries pushes down LIMIT n
 /// where applicable to reduce the amount of scanned / processed data
@@ -43,25 +42,6 @@ fn limit_push_down(
     execution_props: &ExecutionProps,
 ) -> Result<LogicalPlan> {
     match (plan, upper_limit) {
-        (
-            LogicalPlan::Explain {
-                verbose,
-                schema,
-                plan,
-                stringified_plans,
-            },
-            _,
-        ) => {
-            let schema = schema.as_ref().to_owned().into();
-            optimize_explain(
-                optimizer,
-                *verbose,
-                plan,
-                stringified_plans,
-                &schema,
-                execution_props,
-            )
-        }
         (LogicalPlan::Limit { n, input }, upper_limit) => {
             let smallest = upper_limit.map(|x| std::cmp::min(x, *n)).unwrap_or(*n);
             Ok(LogicalPlan::Limit {
diff --git a/datafusion/src/optimizer/projection_push_down.rs b/datafusion/src/optimizer/projection_push_down.rs
index 089dca2..0de36f3 100644
--- a/datafusion/src/optimizer/projection_push_down.rs
+++ b/datafusion/src/optimizer/projection_push_down.rs
@@ -18,7 +18,7 @@
 //! Projection Push Down optimizer rule ensures that only referenced columns are
 //! loaded into memory
 
-use crate::error::Result;
+use crate::error::{DataFusionError, Result};
 use crate::execution::context::ExecutionProps;
 use crate::logical_plan::{
     build_join_schema, Column, DFField, DFSchema, DFSchemaRef, LogicalPlan,
@@ -33,7 +33,6 @@ use std::{
     collections::{BTreeSet, HashSet},
     sync::Arc,
 };
-use utils::optimize_explain;
 
 /// Optimizer that removes unused projections and aggregations from plans
 /// This reduces both scans and
@@ -354,22 +353,9 @@ fn optimize_plan(
                 limit: *limit,
             })
         }
-        LogicalPlan::Explain {
-            verbose,
-            plan,
-            stringified_plans,
-            schema,
-        } => {
-            let schema = schema.as_ref().to_owned().into();
-            optimize_explain(
-                optimizer,
-                *verbose,
-                &*plan,
-                stringified_plans,
-                &schema,
-                execution_props,
-            )
-        }
+        LogicalPlan::Explain { .. } => Err(DataFusionError::Internal(
+            "Unsupported logical plan: Explain must be root of the plan".to_string(),
+        )),
         LogicalPlan::Union {
             inputs,
             schema,
diff --git a/datafusion/src/optimizer/simplify_expressions.rs b/datafusion/src/optimizer/simplify_expressions.rs
index 0e65de0..f629eaf 100644
--- a/datafusion/src/optimizer/simplify_expressions.rs
+++ b/datafusion/src/optimizer/simplify_expressions.rs
@@ -22,7 +22,6 @@ use crate::logical_plan::LogicalPlan;
 use crate::logical_plan::{lit, Expr};
 use crate::optimizer::optimizer::OptimizerRule;
 use crate::optimizer::utils;
-use crate::optimizer::utils::optimize_explain;
 use crate::scalar::ScalarValue;
 use crate::{error::Result, logical_plan::Operator};
 
@@ -278,27 +277,9 @@ impl OptimizerRule for SimplifyExpressions {
     fn optimize(
         &self,
         plan: &LogicalPlan,
-        execution_props: &ExecutionProps,
+        _execution_props: &ExecutionProps,
     ) -> Result<LogicalPlan> {
-        match plan {
-            LogicalPlan::Explain {
-                verbose,
-                plan,
-                stringified_plans,
-                schema,
-            } => {
-                let schema = schema.as_ref().to_owned().into();
-                optimize_explain(
-                    self,
-                    *verbose,
-                    &*plan,
-                    stringified_plans,
-                    &schema,
-                    execution_props,
-                )
-            }
-            _ => optimize(plan),
-        }
+        optimize(plan)
     }
 }
 
diff --git a/datafusion/src/optimizer/utils.rs b/datafusion/src/optimizer/utils.rs
index 88380ea..615f0cc 100644
--- a/datafusion/src/optimizer/utils.rs
+++ b/datafusion/src/optimizer/utils.rs
@@ -17,15 +17,11 @@
 
 //! Collection of utility functions that are leveraged by the query optimizer rules
 
-use std::{collections::HashSet, sync::Arc};
-
-use arrow::datatypes::Schema;
-
 use super::optimizer::OptimizerRule;
 use crate::execution::context::ExecutionProps;
 use crate::logical_plan::{
     build_join_schema, Column, DFSchemaRef, Expr, LogicalPlan, LogicalPlanBuilder,
-    Operator, Partitioning, PlanType, Recursion, StringifiedPlan, ToDFSchema,
+    Operator, Partitioning, Recursion,
 };
 use crate::prelude::lit;
 use crate::scalar::ScalarValue;
@@ -33,6 +29,7 @@ use crate::{
     error::{DataFusionError, Result},
     logical_plan::ExpressionVisitor,
 };
+use std::{collections::HashSet, sync::Arc};
 
 const CASE_EXPR_MARKER: &str = "__DATAFUSION_CASE_EXPR__";
 const CASE_ELSE_MARKER: &str = "__DATAFUSION_CASE_ELSE__";
@@ -94,34 +91,6 @@ pub fn expr_to_columns(expr: &Expr, accum: &mut HashSet<Column>) -> Result<()> {
     Ok(())
 }
 
-/// Create a `LogicalPlan::Explain` node by running `optimizer` on the
-/// input plan and capturing the resulting plan string
-pub fn optimize_explain(
-    optimizer: &impl OptimizerRule,
-    verbose: bool,
-    plan: &LogicalPlan,
-    stringified_plans: &[StringifiedPlan],
-    schema: &Schema,
-    execution_props: &ExecutionProps,
-) -> Result<LogicalPlan> {
-    // These are the fields of LogicalPlan::Explain It might be nice
-    // to transform that enum Variant into its own struct and avoid
-    // passing the fields individually
-    let plan = Arc::new(optimizer.optimize(plan, execution_props)?);
-    let mut stringified_plans = stringified_plans.to_vec();
-    let optimizer_name = optimizer.name().into();
-    stringified_plans.push(StringifiedPlan::new(
-        PlanType::OptimizedLogicalPlan { optimizer_name },
-        format!("{:#?}", plan),
-    ));
-    Ok(LogicalPlan::Explain {
-        verbose,
-        plan,
-        stringified_plans,
-        schema: schema.clone().to_dfschema_ref()?,
-    })
-}
-
 /// Convenience rule for writing optimizers: recursively invoke
 /// optimize on plan's children and then return a node of the same
 /// type. Useful for optimizer rules which want to leave the type
@@ -132,23 +101,6 @@ pub fn optimize_children(
     plan: &LogicalPlan,
     execution_props: &ExecutionProps,
 ) -> Result<LogicalPlan> {
-    if let LogicalPlan::Explain {
-        verbose,
-        plan,
-        stringified_plans,
-        schema,
-    } = plan
-    {
-        return optimize_explain(
-            optimizer,
-            *verbose,
-            &*plan,
-            stringified_plans,
-            &schema.as_ref().to_owned().into(),
-            execution_props,
-        );
-    }
-
     let new_exprs = plan.expressions();
     let new_inputs = plan
         .inputs()
@@ -489,7 +441,7 @@ pub fn rewrite_expression(expr: &Expr, expressions: &[Expr]) -> Result<Expr> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::logical_plan::{col, LogicalPlanBuilder};
+    use crate::logical_plan::col;
     use arrow::datatypes::DataType;
     use std::collections::HashSet;
 
@@ -514,61 +466,4 @@ mod tests {
         assert!(accum.contains(&Column::from_name("a")));
         Ok(())
     }
-
-    struct TestOptimizer {}
-
-    impl OptimizerRule for TestOptimizer {
-        fn optimize(
-            &self,
-            plan: &LogicalPlan,
-            _: &ExecutionProps,
-        ) -> Result<LogicalPlan> {
-            Ok(plan.clone())
-        }
-
-        fn name(&self) -> &str {
-            "test_optimizer"
-        }
-    }
-
-    #[test]
-    fn test_optimize_explain() -> Result<()> {
-        let optimizer = TestOptimizer {};
-
-        let empty_plan = LogicalPlanBuilder::empty(false).build()?;
-        let schema = LogicalPlan::explain_schema();
-
-        let optimized_explain = optimize_explain(
-            &optimizer,
-            true,
-            &empty_plan,
-            &[StringifiedPlan::new(PlanType::InitialLogicalPlan, "...")],
-            schema.as_ref(),
-            &ExecutionProps::new(),
-        )?;
-
-        match &optimized_explain {
-            LogicalPlan::Explain {
-                verbose,
-                stringified_plans,
-                ..
-            } => {
-                assert!(*verbose);
-
-                let expected_stringified_plans = vec![
-                    StringifiedPlan::new(PlanType::InitialLogicalPlan, "..."),
-                    StringifiedPlan::new(
-                        PlanType::OptimizedLogicalPlan {
-                            optimizer_name: "test_optimizer".into(),
-                        },
-                        "EmptyRelation",
-                    ),
-                ];
-                assert_eq!(*stringified_plans, expected_stringified_plans);
-            }
-            _ => panic!("Expected explain plan but got {:?}", optimized_explain),
-        }
-
-        Ok(())
-    }
 }
diff --git a/datafusion/src/physical_plan/display.rs b/datafusion/src/physical_plan/display.rs
index 8498e02..e251e4e 100644
--- a/datafusion/src/physical_plan/display.rs
+++ b/datafusion/src/physical_plan/display.rs
@@ -21,6 +21,8 @@
 
 use std::fmt;
 
+use crate::logical_plan::{StringifiedPlan, ToStringifiedPlan};
+
 use super::{accept, ExecutionPlan, ExecutionPlanVisitor};
 
 /// Options for controlling how each [`ExecutionPlan`] should format itself
@@ -131,3 +133,12 @@ impl<'a, 'b> ExecutionPlanVisitor for IndentVisitor<'a, 'b> {
         Ok(true)
     }
 }
+
+impl<'a> ToStringifiedPlan for DisplayableExecutionPlan<'a> {
+    fn to_stringified(
+        &self,
+        plan_type: crate::logical_plan::PlanType,
+    ) -> StringifiedPlan {
+        StringifiedPlan::new(plan_type, self.indent().to_string())
+    }
+}
diff --git a/datafusion/src/physical_plan/explain.rs b/datafusion/src/physical_plan/explain.rs
index 195a7a5..a6a34f5 100644
--- a/datafusion/src/physical_plan/explain.rs
+++ b/datafusion/src/physical_plan/explain.rs
@@ -115,9 +115,20 @@ impl ExecutionPlan for ExplainExec {
             .iter()
             .filter(|s| s.should_display(self.verbose));
 
+        // Identify plans that are not changed
+        let mut prev: Option<&StringifiedPlan> = None;
+
         for p in plans_to_print {
             type_builder.append_value(p.plan_type.to_string())?;
-            plan_builder.append_value(&*p.plan)?;
+            match prev {
+                Some(prev) if !should_show(prev, p) => {
+                    plan_builder.append_value("SAME TEXT AS ABOVE")?;
+                }
+                Some(_) | None => {
+                    plan_builder.append_value(&*p.plan)?;
+                }
+            }
+            prev = Some(p);
         }
 
         let record_batch = RecordBatch::try_new(
@@ -146,3 +157,14 @@ impl ExecutionPlan for ExplainExec {
         }
     }
 }
+
+/// If this plan should be shown, given the previous plan that was
+/// displayed.
+///
+/// This is meant to avoid repeating the same plan over and over again
+/// in explain plans to make clear what is changing
+fn should_show(previous_plan: &StringifiedPlan, this_plan: &StringifiedPlan) -> bool {
+    // if the plans are different, or if they would have been
+    // displayed in the normal explain (aka non verbose) plan
+    (previous_plan.plan != this_plan.plan) || this_plan.should_display(false)
+}
diff --git a/datafusion/src/physical_plan/planner.rs b/datafusion/src/physical_plan/planner.rs
index 5163e4b..e662821 100644
--- a/datafusion/src/physical_plan/planner.rs
+++ b/datafusion/src/physical_plan/planner.rs
@@ -24,9 +24,10 @@ use super::{
 use crate::execution::context::ExecutionContextState;
 use crate::logical_plan::{
     unnormalize_cols, DFSchema, Expr, LogicalPlan, Operator,
-    Partitioning as LogicalPartitioning, PlanType, StringifiedPlan,
+    Partitioning as LogicalPartitioning, PlanType, ToStringifiedPlan,
     UserDefinedLogicalNode,
 };
+use crate::physical_optimizer::optimizer::PhysicalOptimizerRule;
 use crate::physical_plan::explain::ExplainExec;
 use crate::physical_plan::expressions;
 use crate::physical_plan::expressions::{CaseExpr, Column, Literal, PhysicalSortExpr};
@@ -244,7 +245,7 @@ impl PhysicalPlanner for DefaultPhysicalPlanner {
             Some(plan) => Ok(plan),
             None => {
                 let plan = self.create_initial_plan(logical_plan, ctx_state)?;
-                self.optimize_plan(plan, ctx_state)
+                self.optimize_internal(plan, ctx_state, |_, _| {})
             }
         }
     }
@@ -285,23 +286,6 @@ impl DefaultPhysicalPlanner {
         Self { extension_planners }
     }
 
-    /// Optimize a physical plan by applying each physical optimizer
-    fn optimize_plan(
-        &self,
-        plan: Arc<dyn ExecutionPlan>,
-        ctx_state: &ExecutionContextState,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
-        let optimizers = &ctx_state.config.physical_optimizers;
-        debug!("Physical plan:\n{:?}", plan);
-
-        let mut new_plan = plan;
-        for optimizer in optimizers {
-            new_plan = optimizer.optimize(new_plan, &ctx_state.config)?;
-        }
-        debug!("Optimized physical plan:\n{:?}", new_plan);
-        Ok(new_plan)
-    }
-
     /// Create a physical plan from a logical plan
     fn create_initial_plan(
         &self,
@@ -1315,32 +1299,24 @@ impl DefaultPhysicalPlanner {
             schema,
         } = logical_plan
         {
-            let final_logical_plan = StringifiedPlan::new(
-                PlanType::FinalLogicalPlan,
-                plan.display_indent().to_string(),
-            );
+            use PlanType::*;
+            let mut stringified_plans = stringified_plans.clone();
 
-            let input = self.create_initial_plan(plan, ctx_state)?;
+            stringified_plans.push(plan.to_stringified(FinalLogicalPlan));
 
-            let initial_physical_plan = StringifiedPlan::new(
-                PlanType::InitialPhysicalPlan,
-                displayable(input.as_ref()).indent().to_string(),
-            );
+            let input = self.create_initial_plan(plan, ctx_state)?;
 
-            let input = self.optimize_plan(input, ctx_state)?;
+            stringified_plans
+                .push(displayable(input.as_ref()).to_stringified(InitialPhysicalPlan));
 
-            let final_physical_plan = StringifiedPlan::new(
-                PlanType::FinalPhysicalPlan,
-                displayable(input.as_ref()).indent().to_string(),
-            );
+            let input = self.optimize_internal(input, ctx_state, |plan, optimizer| {
+                let optimizer_name = optimizer.name().to_string();
+                let plan_type = OptimizedPhysicalPlan { optimizer_name };
+                stringified_plans.push(displayable(plan).to_stringified(plan_type));
+            })?;
 
-            let stringified_plans = stringified_plans
-                .iter()
-                .cloned()
-                .chain(std::iter::once(final_logical_plan))
-                .chain(std::iter::once(initial_physical_plan))
-                .chain(std::iter::once(final_physical_plan))
-                .collect::<Vec<_>>();
+            stringified_plans
+                .push(displayable(input.as_ref()).to_stringified(FinalPhysicalPlan));
 
             Ok(Some(Arc::new(ExplainExec::new(
                 SchemaRef::new(schema.as_ref().to_owned().into()),
@@ -1351,6 +1327,29 @@ impl DefaultPhysicalPlanner {
             Ok(None)
         }
     }
+
+    /// Optimize a physical plan by applying each physical optimizer,
+    /// calling observer(plan, optimizer after each one)
+    fn optimize_internal<F>(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        ctx_state: &ExecutionContextState,
+        mut observer: F,
+    ) -> Result<Arc<dyn ExecutionPlan>>
+    where
+        F: FnMut(&dyn ExecutionPlan, &dyn PhysicalOptimizerRule),
+    {
+        let optimizers = &ctx_state.config.physical_optimizers;
+        debug!("Physical plan:\n{:?}", plan);
+
+        let mut new_plan = plan;
+        for optimizer in optimizers {
+            new_plan = optimizer.optimize(new_plan, &ctx_state.config)?;
+            observer(new_plan.as_ref(), optimizer.as_ref())
+        }
+        debug!("Optimized physical plan:\n{:?}", new_plan);
+        Ok(new_plan)
+    }
 }
 
 fn tuple_err<T, R>(value: (Result<T>, Result<R>)) -> Result<(T, R)> {
@@ -1645,6 +1644,42 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_explain() {
+        let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]);
+
+        let logical_plan =
+            LogicalPlanBuilder::scan_empty(Some("employee"), &schema, None)
+                .unwrap()
+                .explain(true)
+                .unwrap()
+                .build()
+                .unwrap();
+
+        let plan = plan(&logical_plan).unwrap();
+        if let Some(plan) = plan.as_any().downcast_ref::<ExplainExec>() {
+            let stringified_plans = plan.stringified_plans();
+            assert!(stringified_plans.len() >= 4);
+            assert!(stringified_plans
+                .iter()
+                .any(|p| matches!(p.plan_type, PlanType::FinalLogicalPlan)));
+            assert!(stringified_plans
+                .iter()
+                .any(|p| matches!(p.plan_type, PlanType::InitialPhysicalPlan)));
+            assert!(stringified_plans
+                .iter()
+                .any(|p| matches!(p.plan_type, PlanType::OptimizedPhysicalPlan { .. })));
+            assert!(stringified_plans
+                .iter()
+                .any(|p| matches!(p.plan_type, PlanType::FinalPhysicalPlan)));
+        } else {
+            panic!(
+                "Plan was not an explain plan: {}",
+                displayable(plan.as_ref()).indent()
+            );
+        }
+    }
+
     /// An example extension node that doesn't do anything
     struct NoOpExtensionNode {
         schema: DFSchemaRef,
diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs
index a4bb02c..fa2b035 100644
--- a/datafusion/src/sql/planner.rs
+++ b/datafusion/src/sql/planner.rs
@@ -28,8 +28,8 @@ use crate::logical_plan::window_frames::{WindowFrame, WindowFrameUnits};
 use crate::logical_plan::Expr::Alias;
 use crate::logical_plan::{
     and, builder::expand_wildcard, col, lit, normalize_col, union_with_alias, Column,
-    DFSchema, Expr, LogicalPlan, LogicalPlanBuilder, Operator, PlanType, StringifiedPlan,
-    ToDFSchema,
+    DFSchema, Expr, LogicalPlan, LogicalPlanBuilder, Operator, PlanType, ToDFSchema,
+    ToStringifiedPlan,
 };
 use crate::prelude::JoinType;
 use crate::scalar::ScalarValue;
@@ -233,10 +233,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
     ) -> Result<LogicalPlan> {
         let plan = self.sql_statement_to_plan(statement)?;
 
-        let stringified_plans = vec![StringifiedPlan::new(
-            PlanType::InitialLogicalPlan,
-            plan.display_indent().to_string(),
-        )];
+        let stringified_plans = vec![plan.to_stringified(PlanType::InitialLogicalPlan)];
 
         let schema = LogicalPlan::explain_schema();
         let plan = Arc::new(plan);
diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs
index 95b5596..0ef8b4c 100644
--- a/datafusion/tests/sql.rs
+++ b/datafusion/tests/sql.rs
@@ -2205,6 +2205,13 @@ async fn csv_explain_verbose() {
         "Actual: '{}'",
         actual
     );
+
+    // ensure the "same text as above" optimization is working
+    assert!(
+        actual.contains("SAME TEXT AS ABOVE"),
+        "Actual 2: '{}'",
+        actual
+    );
 }
 
 #[tokio::test]
diff --git a/datafusion/tests/user_defined_plan.rs b/datafusion/tests/user_defined_plan.rs
index e1f8c76..c1269d9 100644
--- a/datafusion/tests/user_defined_plan.rs
+++ b/datafusion/tests/user_defined_plan.rs
@@ -163,9 +163,9 @@ async fn topk_plan() -> Result<()> {
     let mut ctx = setup_table(make_topk_context()).await?;
 
     let expected = vec![
-        "| logical_plan after topk                 | TopK: k=3                                                                                |",
-        "|                                         |   Projection: #sales.customer_id, #sales.revenue                                         |",
-        "|                                         |     TableScan: sales projection=Some([0, 1])                                             |",
+        "| logical_plan after topk                   | TopK: k=3                                                                                |",
+        "|                                           |   Projection: #sales.customer_id, #sales.revenue                                         |",
+        "|                                           |     TableScan: sales projection=Some([0, 1])                                             |",
     ].join("\n");
 
     let explain_query = format!("EXPLAIN VERBOSE {}", QUERY);