You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2023/09/18 14:07:01 UTC
[arrow-datafusion] branch main updated: Implement protobuf serialization for AnalyzeExec (#7574)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 938505b8f7 Implement protobuf serialization for AnalyzeExec (#7574)
938505b8f7 is described below
commit 938505b8f7120353d8b933908822f3be048762bc
Author: Adhish Singla <ad...@gmail.com>
AuthorDate: Mon Sep 18 19:36:54 2023 +0530
Implement protobuf serialization for AnalyzeExec (#7574)
* feat: serde for AnalyzeExec
* add test for AnalyzeExec serde
* fix for failing pipeline
* box the AnalyzeExec
---
datafusion/physical-plan/src/analyze.rs | 15 +++
datafusion/proto/proto/datafusion.proto | 8 ++
datafusion/proto/src/generated/pbjson.rs | 156 ++++++++++++++++++++++++++++++
datafusion/proto/src/generated/prost.rs | 16 ++-
datafusion/proto/src/physical_plan/mod.rs | 48 ++++++++-
5 files changed, 241 insertions(+), 2 deletions(-)
diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs
index b7e95c8b1b..3a2cac59cf 100644
--- a/datafusion/physical-plan/src/analyze.rs
+++ b/datafusion/physical-plan/src/analyze.rs
@@ -62,6 +62,21 @@ impl AnalyzeExec {
schema,
}
}
+
+ /// access to verbose
+ pub fn verbose(&self) -> bool {
+ self.verbose
+ }
+
+ /// access to show_statistics
+ pub fn show_statistics(&self) -> bool {
+ self.show_statistics
+ }
+
+ /// The input plan
+ pub fn input(&self) -> &Arc<dyn ExecutionPlan> {
+ &self.input
+ }
}
impl DisplayAs for AnalyzeExec {
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index a0148c8af0..f036d6f447 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -1110,6 +1110,7 @@ message PhysicalPlanNode {
ExplainExecNode explain = 20;
SortPreservingMergeExecNode sort_preserving_merge = 21;
NestedLoopJoinExecNode nested_loop_join = 22;
+ AnalyzeExecNode analyze = 23;
}
}
@@ -1345,6 +1346,13 @@ message ExplainExecNode {
bool verbose = 3;
}
+message AnalyzeExecNode {
+ bool verbose = 1;
+ bool show_statistics = 2;
+ PhysicalPlanNode input = 3;
+ Schema schema = 4;
+}
+
message CrossJoinExecNode {
PhysicalPlanNode left = 1;
PhysicalPlanNode right = 2;
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index 9485161174..a70de83342 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -1065,6 +1065,149 @@ impl<'de> serde::Deserialize<'de> for AliasNode {
deserializer.deserialize_struct("datafusion.AliasNode", FIELDS, GeneratedVisitor)
}
}
+impl serde::Serialize for AnalyzeExecNode {
+ #[allow(deprecated)]
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ use serde::ser::SerializeStruct;
+ let mut len = 0;
+ if self.verbose {
+ len += 1;
+ }
+ if self.show_statistics {
+ len += 1;
+ }
+ if self.input.is_some() {
+ len += 1;
+ }
+ if self.schema.is_some() {
+ len += 1;
+ }
+ let mut struct_ser = serializer.serialize_struct("datafusion.AnalyzeExecNode", len)?;
+ if self.verbose {
+ struct_ser.serialize_field("verbose", &self.verbose)?;
+ }
+ if self.show_statistics {
+ struct_ser.serialize_field("showStatistics", &self.show_statistics)?;
+ }
+ if let Some(v) = self.input.as_ref() {
+ struct_ser.serialize_field("input", v)?;
+ }
+ if let Some(v) = self.schema.as_ref() {
+ struct_ser.serialize_field("schema", v)?;
+ }
+ struct_ser.end()
+ }
+}
+impl<'de> serde::Deserialize<'de> for AnalyzeExecNode {
+ #[allow(deprecated)]
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ const FIELDS: &[&str] = &[
+ "verbose",
+ "show_statistics",
+ "showStatistics",
+ "input",
+ "schema",
+ ];
+
+ #[allow(clippy::enum_variant_names)]
+ enum GeneratedField {
+ Verbose,
+ ShowStatistics,
+ Input,
+ Schema,
+ }
+ impl<'de> serde::Deserialize<'de> for GeneratedField {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ struct GeneratedVisitor;
+
+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+ type Value = GeneratedField;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(formatter, "expected one of: {:?}", &FIELDS)
+ }
+
+ #[allow(unused_variables)]
+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
+ where
+ E: serde::de::Error,
+ {
+ match value {
+ "verbose" => Ok(GeneratedField::Verbose),
+ "showStatistics" | "show_statistics" => Ok(GeneratedField::ShowStatistics),
+ "input" => Ok(GeneratedField::Input),
+ "schema" => Ok(GeneratedField::Schema),
+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
+ }
+ }
+ }
+ deserializer.deserialize_identifier(GeneratedVisitor)
+ }
+ }
+ struct GeneratedVisitor;
+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+ type Value = AnalyzeExecNode;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str("struct datafusion.AnalyzeExecNode")
+ }
+
+ fn visit_map<V>(self, mut map: V) -> std::result::Result<AnalyzeExecNode, V::Error>
+ where
+ V: serde::de::MapAccess<'de>,
+ {
+ let mut verbose__ = None;
+ let mut show_statistics__ = None;
+ let mut input__ = None;
+ let mut schema__ = None;
+ while let Some(k) = map.next_key()? {
+ match k {
+ GeneratedField::Verbose => {
+ if verbose__.is_some() {
+ return Err(serde::de::Error::duplicate_field("verbose"));
+ }
+ verbose__ = Some(map.next_value()?);
+ }
+ GeneratedField::ShowStatistics => {
+ if show_statistics__.is_some() {
+ return Err(serde::de::Error::duplicate_field("showStatistics"));
+ }
+ show_statistics__ = Some(map.next_value()?);
+ }
+ GeneratedField::Input => {
+ if input__.is_some() {
+ return Err(serde::de::Error::duplicate_field("input"));
+ }
+ input__ = map.next_value()?;
+ }
+ GeneratedField::Schema => {
+ if schema__.is_some() {
+ return Err(serde::de::Error::duplicate_field("schema"));
+ }
+ schema__ = map.next_value()?;
+ }
+ }
+ }
+ Ok(AnalyzeExecNode {
+ verbose: verbose__.unwrap_or_default(),
+ show_statistics: show_statistics__.unwrap_or_default(),
+ input: input__,
+ schema: schema__,
+ })
+ }
+ }
+ deserializer.deserialize_struct("datafusion.AnalyzeExecNode", FIELDS, GeneratedVisitor)
+ }
+}
impl serde::Serialize for AnalyzeNode {
#[allow(deprecated)]
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
@@ -16354,6 +16497,9 @@ impl serde::Serialize for PhysicalPlanNode {
physical_plan_node::PhysicalPlanType::NestedLoopJoin(v) => {
struct_ser.serialize_field("nestedLoopJoin", v)?;
}
+ physical_plan_node::PhysicalPlanType::Analyze(v) => {
+ struct_ser.serialize_field("analyze", v)?;
+ }
}
}
struct_ser.end()
@@ -16397,6 +16543,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
"sortPreservingMerge",
"nested_loop_join",
"nestedLoopJoin",
+ "analyze",
];
#[allow(clippy::enum_variant_names)]
@@ -16422,6 +16569,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
Explain,
SortPreservingMerge,
NestedLoopJoin,
+ Analyze,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -16464,6 +16612,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
"explain" => Ok(GeneratedField::Explain),
"sortPreservingMerge" | "sort_preserving_merge" => Ok(GeneratedField::SortPreservingMerge),
"nestedLoopJoin" | "nested_loop_join" => Ok(GeneratedField::NestedLoopJoin),
+ "analyze" => Ok(GeneratedField::Analyze),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -16631,6 +16780,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {
return Err(serde::de::Error::duplicate_field("nestedLoopJoin"));
}
physical_plan_type__ = map.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::NestedLoopJoin)
+;
+ }
+ GeneratedField::Analyze => {
+ if physical_plan_type__.is_some() {
+ return Err(serde::de::Error::duplicate_field("analyze"));
+ }
+ physical_plan_type__ = map.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::Analyze)
;
}
}
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index 22a4beadb8..f90f462024 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -1452,7 +1452,7 @@ pub mod owned_table_reference {
pub struct PhysicalPlanNode {
#[prost(
oneof = "physical_plan_node::PhysicalPlanType",
- tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22"
+ tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23"
)]
pub physical_plan_type: ::core::option::Option<physical_plan_node::PhysicalPlanType>,
}
@@ -1505,6 +1505,8 @@ pub mod physical_plan_node {
),
#[prost(message, tag = "22")]
NestedLoopJoin(::prost::alloc::boxed::Box<super::NestedLoopJoinExecNode>),
+ #[prost(message, tag = "23")]
+ Analyze(::prost::alloc::boxed::Box<super::AnalyzeExecNode>),
}
}
#[allow(clippy::derive_partial_eq_without_eq)]
@@ -1882,6 +1884,18 @@ pub struct ExplainExecNode {
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct AnalyzeExecNode {
+ #[prost(bool, tag = "1")]
+ pub verbose: bool,
+ #[prost(bool, tag = "2")]
+ pub show_statistics: bool,
+ #[prost(message, optional, boxed, tag = "3")]
+ pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,
+ #[prost(message, optional, tag = "4")]
+ pub schema: ::core::option::Option<Schema>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CrossJoinExecNode {
#[prost(message, optional, boxed, tag = "1")]
pub left: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 3b9564da60..88558d1be2 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -27,6 +27,7 @@ use datafusion::execution::FunctionRegistry;
use datafusion::logical_expr::WindowFrame;
use datafusion::physical_plan::aggregates::{create_aggregate_expr, AggregateMode};
use datafusion::physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};
+use datafusion::physical_plan::analyze::AnalyzeExec;
use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion::physical_plan::empty::EmptyExec;
@@ -778,6 +779,20 @@ impl AsExecutionPlan for PhysicalPlanNode {
&join_type.into(),
)?))
}
+ PhysicalPlanType::Analyze(analyze) => {
+ let input: Arc<dyn ExecutionPlan> = into_physical_plan(
+ &analyze.input,
+ registry,
+ runtime,
+ extension_codec,
+ )?;
+ Ok(Arc::new(AnalyzeExec::new(
+ analyze.verbose,
+ analyze.show_statistics,
+ input,
+ Arc::new(analyze.schema.as_ref().unwrap().try_into()?),
+ )))
+ }
}
}
@@ -825,6 +840,21 @@ impl AsExecutionPlan for PhysicalPlanNode {
},
))),
})
+ } else if let Some(exec) = plan.downcast_ref::<AnalyzeExec>() {
+ let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
+ exec.input().to_owned(),
+ extension_codec,
+ )?;
+ Ok(protobuf::PhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::Analyze(Box::new(
+ protobuf::AnalyzeExecNode {
+ verbose: exec.verbose(),
+ show_statistics: exec.show_statistics(),
+ input: Some(Box::new(input)),
+ schema: Some(exec.schema().as_ref().try_into()?),
+ },
+ ))),
+ })
} else if let Some(exec) = plan.downcast_ref::<FilterExec>() {
let input = protobuf::PhysicalPlanNode::try_from_physical_plan(
exec.input().to_owned(),
@@ -1391,6 +1421,7 @@ mod roundtrip_tests {
use datafusion::physical_expr::expressions::{cast, in_list};
use datafusion::physical_expr::ScalarFunctionExpr;
use datafusion::physical_plan::aggregates::PhysicalGroupBy;
+ use datafusion::physical_plan::analyze::AnalyzeExec;
use datafusion::physical_plan::expressions::{like, BinaryExpr, GetIndexedFieldExpr};
use datafusion::physical_plan::functions::make_scalar_function;
use datafusion::physical_plan::projection::ProjectionExec;
@@ -1946,7 +1977,7 @@ mod roundtrip_tests {
];
let schema = Schema::new(fields);
- let input = Arc::new(EmptyExec::new(false, Arc::new(schema.clone())));
+ let input = Arc::new(EmptyExec::new(true, Arc::new(schema.clone())));
let col_arg = col("arg", &schema)?;
let col_key = col("key", &schema)?;
@@ -1993,4 +2024,19 @@ mod roundtrip_tests {
roundtrip_test(plan)
}
+
+ #[test]
+ fn rountrip_analyze() -> Result<()> {
+ let field_a = Field::new("plan_type", DataType::Utf8, false);
+ let field_b = Field::new("plan", DataType::Utf8, false);
+ let schema = Schema::new(vec![field_a, field_b]);
+ let input = Arc::new(EmptyExec::new(true, Arc::new(schema.clone())));
+
+ roundtrip_test(Arc::new(AnalyzeExec::new(
+ false,
+ false,
+ input,
+ Arc::new(schema),
+ )))
+ }
}