You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2023/09/18 14:07:01 UTC

[arrow-datafusion] branch main updated: Implement protobuf serialization for AnalyzeExec (#7574)

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 938505b8f7 Implement protobuf serialization for AnalyzeExec (#7574)
938505b8f7 is described below

commit 938505b8f7120353d8b933908822f3be048762bc
Author: Adhish Singla <ad...@gmail.com>
AuthorDate: Mon Sep 18 19:36:54 2023 +0530

    Implement protobuf serialization for AnalyzeExec (#7574)
    
    * feat: serde for AnalyzeExec
    
    * add test for AnalyzeExec serde
    
    * fix for failing pipeline
    
    * box the AnalyzeExec
---
 datafusion/physical-plan/src/analyze.rs   |  15 +++
 datafusion/proto/proto/datafusion.proto   |   8 ++
 datafusion/proto/src/generated/pbjson.rs  | 156 ++++++++++++++++++++++++++++++
 datafusion/proto/src/generated/prost.rs   |  16 ++-
 datafusion/proto/src/physical_plan/mod.rs |  48 ++++++++-
 5 files changed, 241 insertions(+), 2 deletions(-)

diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs
index b7e95c8b1b..3a2cac59cf 100644
--- a/datafusion/physical-plan/src/analyze.rs
+++ b/datafusion/physical-plan/src/analyze.rs
@@ -62,6 +62,21 @@ impl AnalyzeExec {
             schema,
         }
     }
+
+    /// access to verbose
+    pub fn verbose(&self) -> bool {
+        self.verbose
+    }
+
+    /// access to show_statistics
+    pub fn show_statistics(&self) -> bool {
+        self.show_statistics
+    }
+
+    /// The input plan
+    pub fn input(&self) -> &Arc<dyn ExecutionPlan> {
+        &self.input
+    }
 }
 
 impl DisplayAs for AnalyzeExec {
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index a0148c8af0..f036d6f447 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -1110,6 +1110,7 @@ message PhysicalPlanNode {
     ExplainExecNode explain = 20;
     SortPreservingMergeExecNode sort_preserving_merge = 21;
     NestedLoopJoinExecNode nested_loop_join = 22;
+    AnalyzeExecNode analyze = 23;
   }
 }
 
@@ -1345,6 +1346,13 @@ message ExplainExecNode {
   bool verbose = 3;
 }
 
+message AnalyzeExecNode {
+  bool verbose = 1;
+  bool show_statistics = 2;
+  PhysicalPlanNode input = 3;
+  Schema schema = 4;
+}
+
 message CrossJoinExecNode {
   PhysicalPlanNode left = 1;
   PhysicalPlanNode right = 2;
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index 9485161174..a70de83342 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -1065,6 +1065,149 @@ impl<'de> serde::Deserialize<'de> for AliasNode {
         deserializer.deserialize_struct("datafusion.AliasNode", FIELDS, GeneratedVisitor)
     }
 }
+impl serde::Serialize for AnalyzeExecNode {
+    #[allow(deprecated)]
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        use serde::ser::SerializeStruct;
+        let mut len = 0;
+        if self.verbose {
+            len += 1;
+        }
+        if self.show_statistics {
+            len += 1;
+        }
+        if self.input.is_some() {
+            len += 1;
+        }
+        if self.schema.is_some() {
+            len += 1;
+        }
+        let mut struct_ser = serializer.serialize_struct("datafusion.AnalyzeExecNode", len)?;
+        if self.verbose {
+            struct_ser.serialize_field("verbose", &self.verbose)?;
+        }
+        if self.show_statistics {
+            struct_ser.serialize_field("showStatistics", &self.show_statistics)?;
+        }
+        if let Some(v) = self.input.as_ref() {
+            struct_ser.serialize_field("input", v)?;
+        }
+        if let Some(v) = self.schema.as_ref() {
+            struct_ser.serialize_field("schema", v)?;
+        }
+        struct_ser.end()
+    }
+}
+impl<'de> serde::Deserialize<'de> for AnalyzeExecNode {
+    #[allow(deprecated)]
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        const FIELDS: &[&str] = &[
+            "verbose",
+            "show_statistics",
+            "showStatistics",
+            "input",
+            "schema",
+        ];
+
+        #[allow(clippy::enum_variant_names)]
+        enum GeneratedField {
+            Verbose,
+            ShowStatistics,
+            Input,
+            Schema,
+        }
+        impl<'de> serde::Deserialize<'de> for GeneratedField {
+            fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
+            where
+                D: serde::Deserializer<'de>,
+            {
+                struct GeneratedVisitor;
+
+                impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+                    type Value = GeneratedField;
+
+                    fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                        write!(formatter, "expected one of: {:?}", &FIELDS)
+                    }
+
+                    #[allow(unused_variables)]
+                    fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
+                    where
+                        E: serde::de::Error,
+                    {
+                        match value {
+                            "verbose" => Ok(GeneratedField::Verbose),
+                            "showStatistics" | "show_statistics" => Ok(GeneratedField::ShowStatistics),
+                            "input" => Ok(GeneratedField::Input),
+                            "schema" => Ok(GeneratedField::Schema),
+                            _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
+                        }
+                    }
+                }
+                deserializer.deserialize_identifier(GeneratedVisitor)
+            }
+        }
+        struct GeneratedVisitor;
+        impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+            type Value = AnalyzeExecNode;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                formatter.write_str("struct datafusion.AnalyzeExecNode")
+            }
+
+            fn visit_map<V>(self, mut map: V) -> std::result::Result<AnalyzeExecNode, V::Error>
+                where
+                    V: serde::de::MapAccess<'de>,
+            {
+                let mut verbose__ = None;
+                let mut show_statistics__ = None;
+                let mut input__ = None;
+                let mut schema__ = None;
+                while let Some(k) = map.next_key()? {
+                    match k {
+                        GeneratedField::Verbose => {
+                            if verbose__.is_some() {
+                                return Err(serde::de::Error::duplicate_field("verbose"));
+                            }
+                            verbose__ = Some(map.next_value()?);
+                        }
+                        GeneratedField::ShowStatistics => {
+                            if show_statistics__.is_some() {
+                                return Err(serde::de::Error::duplicate_field("showStatistics"));
+                            }
+                            show_statistics__ = Some(map.next_value()?);
+                        }
+                        GeneratedField::Input => {
+                            if input__.is_some() {
+                                return Err(serde::de::Error::duplicate_field("input"));
+                            }
+                            input__ = map.next_value()?;
+                        }
+                        GeneratedField::Schema => {
+                            if schema__.is_some() {
+                                return Err(serde::de::Error::duplicate_field("schema"));
+                            }
+                            schema__ = map.next_value()?;
+                        }
+                    }
+                }
+                Ok(AnalyzeExecNode {
+                    verbose: verbose__.unwrap_or_default(),
+                    show_statistics: show_statistics__.unwrap_or_default(),
+                    input: input__,
+                    schema: schema__,
+                })
+            }
+        }
+        deserializer.deserialize_struct("datafusion.AnalyzeExecNode", FIELDS, GeneratedVisitor)
+    }
+}
 impl serde::Serialize for AnalyzeNode {
     #[allow(deprecated)]
     fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
@@ -16354,6 +16497,9 @@ impl serde::Serialize for PhysicalPlanNode {
                 physical_plan_node::PhysicalPlanType::NestedLoopJoin(v) => {
                     struct_ser.serialize_field("nestedLoopJoin", v)?;
                 }
+                physical_plan_node::PhysicalPlanType::Analyze(v) => {
+                    struct_ser.serialize_field("analyze", v)?;
+                }
             }
         }
         struct_ser.end()
@@ -16397,6 +16543,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
             "sortPreservingMerge",
             "nested_loop_join",
             "nestedLoopJoin",
+            "analyze",
         ];
 
         #[allow(clippy::enum_variant_names)]
@@ -16422,6 +16569,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
             Explain,
             SortPreservingMerge,
             NestedLoopJoin,
+            Analyze,
         }
         impl<'de> serde::Deserialize<'de> for GeneratedField {
             fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -16464,6 +16612,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
                             "explain" => Ok(GeneratedField::Explain),
                             "sortPreservingMerge" | "sort_preserving_merge" => Ok(GeneratedField::SortPreservingMerge),
                             "nestedLoopJoin" | "nested_loop_join" => Ok(GeneratedField::NestedLoopJoin),
+                            "analyze" => Ok(GeneratedField::Analyze),
                             _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
                         }
                     }
@@ -16631,6 +16780,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
                                 return Err(serde::de::Error::duplicate_field("nestedLoopJoin"));
                             }
                             physical_plan_type__ = map.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::NestedLoopJoin)
+;
+                        }
+                        GeneratedField::Analyze => {
+                            if physical_plan_type__.is_some() {
+                                return Err(serde::de::Error::duplicate_field("analyze"));
+                            }
+                            physical_plan_type__ = map.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::Analyze)
 ;
                         }
                     }
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index 22a4beadb8..f90f462024 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -1452,7 +1452,7 @@ pub mod owned_table_reference {
 pub struct PhysicalPlanNode {
     #[prost(
         oneof = "physical_plan_node::PhysicalPlanType",
-        tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22"
+        tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23"
     )]
     pub physical_plan_type: ::core::option::Option<physical_plan_node::PhysicalPlanType>,
 }
@@ -1505,6 +1505,8 @@ pub mod physical_plan_node {
         ),
         #[prost(message, tag = "22")]
         NestedLoopJoin(::prost::alloc::boxed::Box<super::NestedLoopJoinExecNode>),
+        #[prost(message, tag = "23")]
+        Analyze(::prost::alloc::boxed::Box<super::AnalyzeExecNode>),
     }
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -1882,6 +1884,18 @@ pub struct ExplainExecNode {
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct AnalyzeExecNode {
+    #[prost(bool, tag = "1")]
+    pub verbose: bool,
+    #[prost(bool, tag = "2")]
+    pub show_statistics: bool,
+    #[prost(message, optional, boxed, tag = "3")]
+    pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,
+    #[prost(message, optional, tag = "4")]
+    pub schema: ::core::option::Option<Schema>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct CrossJoinExecNode {
     #[prost(message, optional, boxed, tag = "1")]
     pub left: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 3b9564da60..88558d1be2 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -27,6 +27,7 @@ use datafusion::execution::FunctionRegistry;
 use datafusion::logical_expr::WindowFrame;
 use datafusion::physical_plan::aggregates::{create_aggregate_expr, AggregateMode};
 use datafusion::physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};
+use datafusion::physical_plan::analyze::AnalyzeExec;
 use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
 use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
 use datafusion::physical_plan::empty::EmptyExec;
@@ -778,6 +779,20 @@ impl AsExecutionPlan for PhysicalPlanNode {
                     &join_type.into(),
                 )?))
             }
+            PhysicalPlanType::Analyze(analyze) => {
+                let input: Arc<dyn ExecutionPlan> = into_physical_plan(
+                    &analyze.input,
+                    registry,
+                    runtime,
+                    extension_codec,
+                )?;
+                Ok(Arc::new(AnalyzeExec::new(
+                    analyze.verbose,
+                    analyze.show_statistics,
+                    input,
+                    Arc::new(analyze.schema.as_ref().unwrap().try_into()?),
+                )))
+            }
         }
     }
 
@@ -825,6 +840,21 @@ impl AsExecutionPlan for PhysicalPlanNode {
                     },
                 ))),
             })
+        } else if let Some(exec) = plan.downcast_ref::<AnalyzeExec>() {
+            let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+                exec.input().to_owned(),
+                extension_codec,
+            )?;
+            Ok(protobuf::PhysicalPlanNode {
+                physical_plan_type: Some(PhysicalPlanType::Analyze(Box::new(
+                    protobuf::AnalyzeExecNode {
+                        verbose: exec.verbose(),
+                        show_statistics: exec.show_statistics(),
+                        input: Some(Box::new(input)),
+                        schema: Some(exec.schema().as_ref().try_into()?),
+                    },
+                ))),
+            })
         } else if let Some(exec) = plan.downcast_ref::<FilterExec>() {
             let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
                 exec.input().to_owned(),
@@ -1391,6 +1421,7 @@ mod roundtrip_tests {
     use datafusion::physical_expr::expressions::{cast, in_list};
     use datafusion::physical_expr::ScalarFunctionExpr;
     use datafusion::physical_plan::aggregates::PhysicalGroupBy;
+    use datafusion::physical_plan::analyze::AnalyzeExec;
     use datafusion::physical_plan::expressions::{like, BinaryExpr, GetIndexedFieldExpr};
     use datafusion::physical_plan::functions::make_scalar_function;
     use datafusion::physical_plan::projection::ProjectionExec;
@@ -1946,7 +1977,7 @@ mod roundtrip_tests {
         ];
 
         let schema = Schema::new(fields);
-        let input = Arc::new(EmptyExec::new(false, Arc::new(schema.clone())));
+        let input = Arc::new(EmptyExec::new(true, Arc::new(schema.clone())));
 
         let col_arg = col("arg", &schema)?;
         let col_key = col("key", &schema)?;
@@ -1993,4 +2024,19 @@ mod roundtrip_tests {
 
         roundtrip_test(plan)
     }
+
+    #[test]
+    fn rountrip_analyze() -> Result<()> {
+        let field_a = Field::new("plan_type", DataType::Utf8, false);
+        let field_b = Field::new("plan", DataType::Utf8, false);
+        let schema = Schema::new(vec![field_a, field_b]);
+        let input = Arc::new(EmptyExec::new(true, Arc::new(schema.clone())));
+
+        roundtrip_test(Arc::new(AnalyzeExec::new(
+            false,
+            false,
+            input,
+            Arc::new(schema),
+        )))
+    }
 }