You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/04 03:08:23 UTC
(datafusion) 01/01: fix(proto): check integer conversions in common options and constraints (#24726)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24726-e1942b1446e437515332ba8f09ce6b0cd6f46e75
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 54f5cf162c3b1671915f343d9a79d31566ea3e28
Author: Burak Şen <bu...@gmail.com>
AuthorDate: Fri Sep 4 03:08:02 2026 +0000
fix(proto): check integer conversions in common options and constraints (#24726)
## Which issue does this PR close?
- Closes #24170.
- Follow-up to #24483.
## Rationale for this change
#24483 added checked integer conversions across logical and physical
plan serialization. Common protobuf conversions for file-format options
and constraints still used unchecked casts, allowing oversized values to
silently truncate on 32-bit targets.
## What changes are included in this PR?
- Apply the checked conversion helper from #24483 to common options and
constraints.
- Make affected conversions fallible, propagating errors instead of
truncating or panicking.
- Update conversion coverage and the 56.0.0 upgrade guide.
The protobuf wire format is unchanged.
## Are these changes tested?
Yes. Existing option round-trip tests pass, and tests cover the fallible
public conversion contracts and malformed constraints.
## Are there any user-facing changes?
Oversized protobuf values now return an error instead of silently
truncating.
Some public `From` conversions are now `TryFrom`, so downstream callers
must handle the resulting error. This migration is documented in the
56.0.0 upgrade guide.
---
datafusion/datasource-csv/src/file_format.rs | 2 +-
datafusion/datasource-json/src/file_format.rs | 2 +-
datafusion/proto-common/src/from_proto/mod.rs | 233 +++++++++++++++------
datafusion/proto-models/src/from_proto.rs | 124 +++++++----
datafusion/proto/src/logical_plan/file_formats.rs | 4 +-
datafusion/proto/src/logical_plan/mod.rs | 2 +-
datafusion/proto/tests/cases/public_conversions.rs | 24 ++-
docs/source/library-user-guide/upgrading/56.0.0.md | 25 +++
8 files changed, 302 insertions(+), 114 deletions(-)
diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs
index f14924563c..a8c27369cb 100644
--- a/datafusion/datasource-csv/src/file_format.rs
+++ b/datafusion/datasource-csv/src/file_format.rs
@@ -941,7 +941,7 @@ impl CsvSink {
/// Encode a [`CsvFormatFactory`]'s options as their protobuf form.
///
-/// The reverse direction is `From<&protobuf::CsvOptions> for CsvOptions` in
+/// The reverse direction is `TryFrom<&protobuf::CsvOptions> for CsvOptions` in
/// `datafusion-proto-models`: `CsvOptions` is a `datafusion-common` type, so
/// that half cannot live here.
#[cfg(feature = "proto")]
diff --git a/datafusion/datasource-json/src/file_format.rs b/datafusion/datasource-json/src/file_format.rs
index 211011f95c..2a9900bc23 100644
--- a/datafusion/datasource-json/src/file_format.rs
+++ b/datafusion/datasource-json/src/file_format.rs
@@ -618,7 +618,7 @@ impl Decoder for JsonDecoder {
/// Encode a [`JsonFormatFactory`]'s options as their protobuf form.
///
-/// The reverse direction is `From<&protobuf::JsonOptions> for JsonOptions` in
+/// The reverse direction is `TryFrom<&protobuf::JsonOptions> for JsonOptions` in
/// `datafusion-proto-models`: `JsonOptions` is a `datafusion-common` type, so
/// that half cannot live here.
#[cfg(feature = "proto")]
diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs
index cf3a3cb75a..7dab16406b 100644
--- a/datafusion/proto-common/src/from_proto/mod.rs
+++ b/datafusion/proto-common/src/from_proto/mod.rs
@@ -46,6 +46,7 @@ use datafusion_common::{
parsers::CompressionTypeVariant,
plan_datafusion_err,
stats::Precision,
+ utils::usize_from_wire,
};
#[derive(Debug)]
@@ -738,30 +739,23 @@ impl From<protobuf::IntervalUnit> for IntervalUnit {
}
}
-impl From<protobuf::Constraints> for Constraints {
- fn from(constraints: protobuf::Constraints) -> Self {
- Constraints::new_unverified(
- constraints
- .constraints
- .into_iter()
- .map(|item| item.into())
- .collect(),
- )
+impl TryFrom<protobuf::Constraints> for Constraints {
+ type Error = DataFusionError;
+
+ fn try_from(
+ constraints: protobuf::Constraints,
+ ) -> datafusion_common::Result<Self, Self::Error> {
+ Self::try_from(&constraints)
}
}
-impl From<protobuf::Constraint> for Constraint {
- fn from(value: protobuf::Constraint) -> Self {
- match value.constraint_mode.unwrap() {
- protobuf::constraint::ConstraintMode::PrimaryKey(elem) => {
- Constraint::PrimaryKey(
- elem.indices.into_iter().map(|item| item as usize).collect(),
- )
- }
- protobuf::constraint::ConstraintMode::Unique(elem) => Constraint::Unique(
- elem.indices.into_iter().map(|item| item as usize).collect(),
- ),
- }
+impl TryFrom<protobuf::Constraint> for Constraint {
+ type Error = DataFusionError;
+
+ fn try_from(
+ value: protobuf::Constraint,
+ ) -> datafusion_common::Result<Self, Self::Error> {
+ Constraint::try_from(&value)
}
}
@@ -884,20 +878,28 @@ impl From<protobuf::JoinSide> for JoinSide {
}
}
-impl From<&protobuf::Constraint> for Constraint {
- fn from(value: &protobuf::Constraint) -> Self {
+impl TryFrom<&protobuf::Constraint> for Constraint {
+ type Error = DataFusionError;
+
+ fn try_from(
+ value: &protobuf::Constraint,
+ ) -> datafusion_common::Result<Self, Self::Error> {
+ let decode_indices = |indices: &[u64]| {
+ indices
+ .iter()
+ .map(|&index| usize_from_wire(index, "Constraint", "indices"))
+ .collect::<datafusion_common::Result<Vec<_>>>()
+ };
match &value.constraint_mode {
Some(protobuf::constraint::ConstraintMode::PrimaryKey(elem)) => {
- Constraint::PrimaryKey(
- elem.indices.iter().map(|&item| item as usize).collect(),
- )
+ Ok(Constraint::PrimaryKey(decode_indices(&elem.indices)?))
}
Some(protobuf::constraint::ConstraintMode::Unique(elem)) => {
- Constraint::Unique(
- elem.indices.iter().map(|&item| item as usize).collect(),
- )
+ Ok(Constraint::Unique(decode_indices(&elem.indices)?))
}
- None => panic!("constraint_mode not set"),
+ None => Err(plan_datafusion_err!(
+ "Constraint: missing required field 'constraint_mode'"
+ )),
}
}
}
@@ -912,8 +914,8 @@ impl TryFrom<&protobuf::Constraints> for Constraints {
constraints
.constraints
.iter()
- .map(|item| item.into())
- .collect(),
+ .map(Constraint::try_from)
+ .collect::<datafusion_common::Result<Vec<_>>>()?,
))
}
}
@@ -1016,7 +1018,10 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions {
newlines_in_values: proto_opts.newlines_in_values.first().map(|h| *h != 0),
compression: proto_opts.compression().into(),
compression_level: proto_opts.compression_level,
- schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),
+ schema_infer_max_rec: proto_opts
+ .schema_infer_max_rec
+ .map(|v| usize_from_wire(v, "CsvOptions", "schema_infer_max_rec"))
+ .transpose()?,
date_format: (!proto_opts.date_format.is_empty())
.then(|| proto_opts.date_format.clone()),
datetime_format: (!proto_opts.datetime_format.is_empty())
@@ -1052,6 +1057,8 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
fn try_from(
value: &protobuf::ParquetOptions,
) -> datafusion_common::Result<Self, Self::Error> {
+ let to_usize =
+ |value: u64, field: &str| usize_from_wire(value, "ParquetOptions", field);
Ok(ParquetOptions {
enable_page_index: value.enable_page_index,
pruning: value.pruning,
@@ -1059,14 +1066,19 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
metadata_size_hint: value
.metadata_size_hint_opt
.map(|opt| match opt {
- protobuf::parquet_options::MetadataSizeHintOpt::MetadataSizeHint(v) => Some(v as usize),
+ protobuf::parquet_options::MetadataSizeHintOpt::MetadataSizeHint(v) => {
+ to_usize(v, "metadata_size_hint")
+ }
})
- .unwrap_or(None),
+ .transpose()?,
pushdown_filters: value.pushdown_filters,
reorder_filters: value.reorder_filters,
force_filter_selections: value.force_filter_selections,
- data_pagesize_limit: value.data_pagesize_limit as usize,
- write_batch_size: value.write_batch_size as usize,
+ data_pagesize_limit: to_usize(
+ value.data_pagesize_limit,
+ "data_pagesize_limit",
+ )?,
+ write_batch_size: to_usize(value.write_batch_size, "write_batch_size")?,
writer_version: value.writer_version.parse().map_err(|e| {
DataFusionError::Internal(format!("Failed to parse writer_version: {e}"))
})?,
@@ -1075,29 +1087,39 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
}).unwrap_or(None),
dictionary_enabled: value.dictionary_enabled_opt.as_ref().map(|protobuf::parquet_options::DictionaryEnabledOpt::DictionaryEnabled(v)| *v),
// Continuing from where we left off in the TryFrom implementation
- dictionary_page_size_limit: value.dictionary_page_size_limit as usize,
+ dictionary_page_size_limit: to_usize(
+ value.dictionary_page_size_limit,
+ "dictionary_page_size_limit",
+ )?,
statistics_enabled: value
.statistics_enabled_opt.clone()
.map(|opt| match opt {
protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v) => Some(v),
})
.unwrap_or(None),
- max_row_group_size: value.max_row_group_size as usize,
- max_in_list_size: value.max_in_list_size as usize,
+ max_row_group_size: to_usize(value.max_row_group_size, "max_row_group_size")?,
+ max_in_list_size: to_usize(value.max_in_list_size, "max_in_list_size")?,
created_by: value.created_by.clone(),
column_index_truncate_length: value
.column_index_truncate_length_opt.as_ref()
.map(|opt| match opt {
- protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v) => Some(*v as usize),
+ protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v) => {
+ to_usize(*v, "column_index_truncate_length")
+ }
})
- .unwrap_or(None),
+ .transpose()?,
statistics_truncate_length: value
.statistics_truncate_length_opt.as_ref()
.map(|opt| match opt {
- protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v) => Some(*v as usize),
+ protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v) => {
+ to_usize(*v, "statistics_truncate_length")
+ }
})
- .unwrap_or(None),
- data_page_row_count_limit: value.data_page_row_count_limit as usize,
+ .transpose()?,
+ data_page_row_count_limit: to_usize(
+ value.data_page_row_count_limit,
+ "data_page_row_count_limit",
+ )?,
encoding: value
.encoding_opt.clone()
.map(|opt| match opt {
@@ -1119,8 +1141,14 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
})
.unwrap_or(None),
allow_single_file_parallelism: value.allow_single_file_parallelism,
- maximum_parallel_row_group_writers: value.maximum_parallel_row_group_writers as usize,
- maximum_buffered_record_batches_per_stream: value.maximum_buffered_record_batches_per_stream as usize,
+ maximum_parallel_row_group_writers: to_usize(
+ value.maximum_parallel_row_group_writers,
+ "maximum_parallel_row_group_writers",
+ )?,
+ maximum_buffered_record_batches_per_stream: to_usize(
+ value.maximum_buffered_record_batches_per_stream,
+ "maximum_buffered_record_batches_per_stream",
+ )?,
schema_force_view_types: value.schema_force_view_types,
binary_as_string: value.binary_as_string,
coerce_int96: value.coerce_int96_opt.clone().map(|opt| match opt {
@@ -1131,24 +1159,34 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
}).unwrap_or(None),
skip_arrow_metadata: value.skip_arrow_metadata,
max_predicate_cache_size: value.max_predicate_cache_size_opt.map(|opt| match opt {
- protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v) => Some(v as usize),
- }).unwrap_or(None),
- max_row_group_bytes: value.max_row_group_bytes_opt.and_then(|opt| match opt {
- protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v) => MaxRowGroupBytes::try_new(v as usize).ok(),
- }),
- content_defined_chunking: value.content_defined_chunking.map(ParquetCdcOptions::from).unwrap_or_default(),
+ protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v) => {
+ to_usize(v, "max_predicate_cache_size")
+ }
+ }).transpose()?,
+ max_row_group_bytes: value.max_row_group_bytes_opt.map(|opt| match opt {
+ protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v) => {
+ MaxRowGroupBytes::try_new(to_usize(v, "max_row_group_bytes")?)
+ }
+ }).transpose()?,
+ content_defined_chunking: value.content_defined_chunking.map(ParquetCdcOptions::try_from).transpose()?.unwrap_or_default(),
})
}
}
-impl From<protobuf::ParquetCdcOptions> for ParquetCdcOptions {
- fn from(value: protobuf::ParquetCdcOptions) -> Self {
- ParquetCdcOptions {
+impl TryFrom<protobuf::ParquetCdcOptions> for ParquetCdcOptions {
+ type Error = DataFusionError;
+
+ fn try_from(
+ value: protobuf::ParquetCdcOptions,
+ ) -> datafusion_common::Result<Self, Self::Error> {
+ let to_usize =
+ |value: u64, field: &str| usize_from_wire(value, "ParquetCdcOptions", field);
+ Ok(ParquetCdcOptions {
enabled: value.enabled,
- min_chunk_size: value.min_chunk_size as usize,
- max_chunk_size: value.max_chunk_size as usize,
+ min_chunk_size: to_usize(value.min_chunk_size, "min_chunk_size")?,
+ max_chunk_size: to_usize(value.max_chunk_size, "max_chunk_size")?,
norm_level: value.norm_level,
- }
+ })
}
}
@@ -1214,9 +1252,9 @@ impl TryFrom<&protobuf::TableParquetOptions> for TableParquetOptions {
global: value
.global
.as_ref()
- .map(|v| v.try_into())
- .unwrap()
- .unwrap(),
+ .map(ParquetOptions::try_from)
+ .transpose()?
+ .unwrap_or_default(),
column_specific_options,
..Default::default()
};
@@ -1234,7 +1272,10 @@ impl TryFrom<&protobuf::JsonOptions> for JsonOptions {
Ok(JsonOptions {
compression: compression.into(),
compression_level: proto_opts.compression_level,
- schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),
+ schema_infer_max_rec: proto_opts
+ .schema_infer_max_rec
+ .map(|v| usize_from_wire(v, "JsonOptions", "schema_infer_max_rec"))
+ .transpose()?,
newline_delimited: proto_opts.newline_delimited.unwrap_or(true),
})
}
@@ -1338,6 +1379,72 @@ mod tests {
MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,
};
+ #[test]
+ fn constraint_requires_mode() {
+ let err = datafusion_common::Constraint::try_from(
+ &crate::protobuf_common::Constraint {
+ constraint_mode: None,
+ },
+ )
+ .unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("Constraint: missing required field 'constraint_mode'")
+ );
+ }
+
+ #[test]
+ fn table_parquet_options_defaults_missing_global() {
+ let recovered = TableParquetOptions::try_from(
+ &crate::protobuf_common::TableParquetOptions::default(),
+ )
+ .expect("missing global should use defaults");
+
+ assert_eq!(recovered, TableParquetOptions::default());
+ }
+
+ #[test]
+ fn table_parquet_options_checks_nested_usize_conversion() {
+ let proto = crate::protobuf_common::TableParquetOptions {
+ global: Some(crate::protobuf_common::ParquetOptions {
+ data_pagesize_limit: u64::MAX,
+ writer_version: "1.0".to_string(),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+ let recovered = TableParquetOptions::try_from(&proto);
+
+ if usize::BITS >= u64::BITS {
+ assert_eq!(recovered.unwrap().global.data_pagesize_limit, usize::MAX);
+ } else {
+ assert_eq!(
+ recovered.unwrap_err().strip_backtrace(),
+ "Error during planning: ParquetOptions: data_pagesize_limit wire value 18446744073709551615 is out of range for usize"
+ );
+ }
+ }
+
+ #[test]
+ fn table_parquet_options_propagates_nested_conversion_error() {
+ let proto = crate::protobuf_common::TableParquetOptions {
+ global: Some(crate::protobuf_common::ParquetOptions {
+ writer_version: "1.0".to_string(),
+ max_row_group_bytes_opt: Some(
+ crate::protobuf_common::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(0),
+ ),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let err = TableParquetOptions::try_from(&proto).unwrap_err();
+ assert_eq!(
+ err.strip_backtrace(),
+ "Invalid or Unsupported Configuration: max_row_group_bytes must be greater than 0"
+ );
+ }
+
fn parquet_options_proto_round_trip(opts: ParquetOptions) -> ParquetOptions {
let proto: crate::protobuf_common::ParquetOptions =
(&opts).try_into().expect("to_proto");
diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rs
index 74ead8c520..16903ec388 100644
--- a/datafusion/proto-models/src/from_proto.rs
+++ b/datafusion/proto-models/src/from_proto.rs
@@ -31,6 +31,7 @@ use datafusion_common::config::{
};
use datafusion_common::display::{PlanType, StringifiedPlan};
use datafusion_common::parsers::{CompressionTypeVariant, CsvQuoteStyle};
+use datafusion_common::utils::usize_from_wire;
use datafusion_common::{
JoinConstraint, JoinType, NullEquality, RecursionUnnestOption, TableReference,
UnnestOptions,
@@ -188,9 +189,11 @@ impl From<protobuf::NullEquality> for NullEquality {
}
}
-impl From<&CsvOptionsProto> for CsvOptions {
- fn from(proto: &CsvOptionsProto) -> Self {
- CsvOptions {
+impl TryFrom<&CsvOptionsProto> for CsvOptions {
+ type Error = datafusion_common::DataFusionError;
+
+ fn try_from(proto: &CsvOptionsProto) -> datafusion_common::Result<Self, Self::Error> {
+ Ok(CsvOptions {
has_header: if !proto.has_header.is_empty() {
Some(proto.has_header[0] != 0)
} else {
@@ -220,7 +223,10 @@ impl From<&CsvOptionsProto> for CsvOptions {
3 => CompressionTypeVariant::ZSTD,
_ => CompressionTypeVariant::UNCOMPRESSED,
},
- schema_infer_max_rec: proto.schema_infer_max_rec.map(|v| v as usize),
+ schema_infer_max_rec: proto
+ .schema_infer_max_rec
+ .map(|value| usize_from_wire(value, "CsvOptions", "schema_infer_max_rec"))
+ .transpose()?,
date_format: if proto.date_format.is_empty() {
None
} else {
@@ -289,13 +295,17 @@ impl From<&CsvOptionsProto> for CsvOptions {
} else {
Some(proto.ignore_trailing_whitespace[0] != 0)
},
- }
+ })
}
}
-impl From<&JsonOptionsProto> for JsonOptions {
- fn from(proto: &JsonOptionsProto) -> Self {
- JsonOptions {
+impl TryFrom<&JsonOptionsProto> for JsonOptions {
+ type Error = datafusion_common::DataFusionError;
+
+ fn try_from(
+ proto: &JsonOptionsProto,
+ ) -> datafusion_common::Result<Self, Self::Error> {
+ Ok(JsonOptions {
compression: match proto.compression {
0 => CompressionTypeVariant::GZIP,
1 => CompressionTypeVariant::BZIP2,
@@ -303,21 +313,32 @@ impl From<&JsonOptionsProto> for JsonOptions {
3 => CompressionTypeVariant::ZSTD,
_ => CompressionTypeVariant::UNCOMPRESSED,
},
- schema_infer_max_rec: proto.schema_infer_max_rec.map(|v| v as usize),
+ schema_infer_max_rec: proto
+ .schema_infer_max_rec
+ .map(|value| {
+ usize_from_wire(value, "JsonOptions", "schema_infer_max_rec")
+ })
+ .transpose()?,
compression_level: proto.compression_level,
newline_delimited: proto.newline_delimited.unwrap_or(true),
- }
+ })
}
}
-impl From<ParquetCdcOptionsProto> for ParquetCdcOptions {
- fn from(value: ParquetCdcOptionsProto) -> Self {
- ParquetCdcOptions {
+impl TryFrom<ParquetCdcOptionsProto> for ParquetCdcOptions {
+ type Error = datafusion_common::DataFusionError;
+
+ fn try_from(
+ value: ParquetCdcOptionsProto,
+ ) -> datafusion_common::Result<Self, Self::Error> {
+ let to_usize =
+ |value: u64, field: &str| usize_from_wire(value, "ParquetCdcOptions", field);
+ Ok(ParquetCdcOptions {
enabled: value.enabled,
- min_chunk_size: value.min_chunk_size as usize,
- max_chunk_size: value.max_chunk_size as usize,
+ min_chunk_size: to_usize(value.min_chunk_size, "min_chunk_size")?,
+ max_chunk_size: to_usize(value.max_chunk_size, "max_chunk_size")?,
norm_level: value.norm_level,
- }
+ })
}
}
@@ -334,6 +355,8 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {
"" => ParquetOptions::default().writer_version,
version => version.parse()?,
};
+ let to_usize =
+ |value: u64, field: &str| usize_from_wire(value, "ParquetOptions", field);
Ok(ParquetOptions {
enable_page_index: proto.enable_page_index,
@@ -344,14 +367,18 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {
.as_ref()
.map(|opt| match opt {
parquet_options::MetadataSizeHintOpt::MetadataSizeHint(size) => {
- *size as usize
+ to_usize(*size, "metadata_size_hint")
}
- }),
+ })
+ .transpose()?,
pushdown_filters: proto.pushdown_filters,
reorder_filters: proto.reorder_filters,
force_filter_selections: proto.force_filter_selections,
- data_pagesize_limit: proto.data_pagesize_limit as usize,
- write_batch_size: proto.write_batch_size as usize,
+ data_pagesize_limit: to_usize(
+ proto.data_pagesize_limit,
+ "data_pagesize_limit",
+ )?,
+ write_batch_size: to_usize(proto.write_batch_size, "write_batch_size")?,
writer_version,
compression: proto.compression_opt.as_ref().map(|opt| match opt {
parquet_options::CompressionOpt::Compression(compression) => {
@@ -365,7 +392,10 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {
) => *enabled,
}
}),
- dictionary_page_size_limit: proto.dictionary_page_size_limit as usize,
+ dictionary_page_size_limit: to_usize(
+ proto.dictionary_page_size_limit,
+ "dictionary_page_size_limit",
+ )?,
statistics_enabled: proto.statistics_enabled_opt.as_ref().map(
|opt| match opt {
parquet_options::StatisticsEnabledOpt::StatisticsEnabled(
@@ -373,22 +403,31 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {
) => statistics.clone(),
},
),
- max_row_group_size: proto.max_row_group_size as usize,
- max_in_list_size: proto.max_in_list_size as usize,
+ max_row_group_size: to_usize(proto.max_row_group_size, "max_row_group_size")?,
+ max_in_list_size: to_usize(proto.max_in_list_size, "max_in_list_size")?,
created_by: proto.created_by.clone(),
column_index_truncate_length: proto
.column_index_truncate_length_opt
.as_ref()
.map(|opt| match opt {
- parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(length) => *length as usize,
- }),
+ parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(length) => {
+ to_usize(*length, "column_index_truncate_length")
+ }
+ })
+ .transpose()?,
statistics_truncate_length: proto
.statistics_truncate_length_opt
.as_ref()
.map(|opt| match opt {
- parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(length) => *length as usize,
- }),
- data_page_row_count_limit: proto.data_page_row_count_limit as usize,
+ parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(length) => {
+ to_usize(*length, "statistics_truncate_length")
+ }
+ })
+ .transpose()?,
+ data_page_row_count_limit: to_usize(
+ proto.data_page_row_count_limit,
+ "data_page_row_count_limit",
+ )?,
encoding: proto.encoding_opt.as_ref().map(|opt| match opt {
parquet_options::EncodingOpt::Encoding(encoding) => {
encoding.clone()
@@ -409,12 +448,14 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {
parquet_options::BloomFilterNdvOpt::BloomFilterNdv(ndv) => *ndv,
}),
allow_single_file_parallelism: proto.allow_single_file_parallelism,
- maximum_parallel_row_group_writers: proto
- .maximum_parallel_row_group_writers
- as usize,
- maximum_buffered_record_batches_per_stream: proto
- .maximum_buffered_record_batches_per_stream
- as usize,
+ maximum_parallel_row_group_writers: to_usize(
+ proto.maximum_parallel_row_group_writers,
+ "maximum_parallel_row_group_writers",
+ )?,
+ maximum_buffered_record_batches_per_stream: to_usize(
+ proto.maximum_buffered_record_batches_per_stream,
+ "maximum_buffered_record_batches_per_stream",
+ )?,
schema_force_view_types: proto.schema_force_view_types,
binary_as_string: proto.binary_as_string,
skip_arrow_metadata: proto.skip_arrow_metadata,
@@ -437,19 +478,22 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {
.map(|opt| match opt {
parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(
size,
- ) => *size as usize,
- }),
+ ) => to_usize(*size, "max_predicate_cache_size"),
+ })
+ .transpose()?,
max_row_group_bytes: proto
.max_row_group_bytes_opt
.as_ref()
- .and_then(|opt| match opt {
+ .map(|opt| match opt {
parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(size) => {
- MaxRowGroupBytes::try_new(*size as usize).ok()
+ MaxRowGroupBytes::try_new(to_usize(*size, "max_row_group_bytes")?)
}
- }),
+ })
+ .transpose()?,
content_defined_chunking: proto
.content_defined_chunking
- .map(ParquetCdcOptions::from)
+ .map(ParquetCdcOptions::try_from)
+ .transpose()?
.unwrap_or_default(),
})
}
diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rs
index 10c54cf55c..7c7a8ef639 100644
--- a/datafusion/proto/src/logical_plan/file_formats.rs
+++ b/datafusion/proto/src/logical_plan/file_formats.rs
@@ -77,7 +77,7 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {
let proto = CsvOptionsProto::decode(buf).map_err(|e| {
exec_datafusion_err!("Failed to decode CsvOptionsProto: {e:?}")
})?;
- let options = CsvOptions::from(&proto);
+ let options = CsvOptions::try_from(&proto)?;
Ok(Arc::new(CsvFormatFactory {
options: Some(options),
}))
@@ -155,7 +155,7 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {
let proto = JsonOptionsProto::decode(buf).map_err(|e| {
exec_datafusion_err!("Failed to decode JsonOptionsProto: {e:?}")
})?;
- let options = JsonOptions::from(&proto);
+ let options = JsonOptions::try_from(&proto)?;
Ok(Arc::new(JsonFormatFactory {
options: Some(options),
}))
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index 90d48b056c..647bceeea1 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -862,7 +862,7 @@ impl AsLogicalPlan for LogicalPlanNode {
.with_definition(definition)
.with_unbounded(create_extern_table.unbounded)
.with_options(create_extern_table.options.clone())
- .with_constraints(constraints.into())
+ .with_constraints(constraints.try_into()?)
.with_column_defaults(column_defaults)
.build(),
),
diff --git a/datafusion/proto/tests/cases/public_conversions.rs b/datafusion/proto/tests/cases/public_conversions.rs
index 1cda8e01a0..ec71960f64 100644
--- a/datafusion/proto/tests/cases/public_conversions.rs
+++ b/datafusion/proto/tests/cases/public_conversions.rs
@@ -35,7 +35,8 @@ use datafusion_common::config::{
};
use datafusion_common::display::StringifiedPlan;
use datafusion_common::{
- JoinConstraint, JoinType, NullEquality, TableReference, UnnestOptions,
+ Constraint, Constraints, DataFusionError, JoinConstraint, JoinType, NullEquality,
+ TableReference, UnnestOptions,
};
use datafusion_datasource::file_groups::FileGroup;
use datafusion_datasource::file_sink_config::FileSinkConfig;
@@ -48,6 +49,7 @@ use datafusion_expr::expr::NullTreatment;
use datafusion_expr::{WindowFrame, WindowFrameBound, WindowFrameUnits};
use datafusion_physical_expr::expressions::Column;
use datafusion_proto::protobuf;
+use datafusion_proto_common::protobuf_common;
/// Asserts `T: From<F>` by naming the conversion.
fn assert_from<F, T: From<F>>() {
@@ -59,6 +61,12 @@ fn assert_try_from<F, T: TryFrom<F>>() {
let _: fn(F) -> Result<T, T::Error> = TryFrom::try_from;
}
+/// Asserts `T: TryFrom<F, Error = DataFusionError>` rather than accepting the
+/// blanket `TryFrom` implementation provided for an infallible `From`.
+fn assert_datafusion_try_from<F, T: TryFrom<F, Error = DataFusionError>>() {
+ let _: fn(F) -> Result<T, DataFusionError> = TryFrom::try_from;
+}
+
#[test]
fn file_scan_conversions_are_std_traits() {
assert_try_from::<&protobuf::PartitionedFile, PartitionedFile>();
@@ -112,16 +120,20 @@ fn common_type_conversions_are_std_traits() {
assert_from::<JoinConstraint, protobuf::JoinConstraint>();
assert_from::<protobuf::NullEquality, NullEquality>();
assert_from::<NullEquality, protobuf::NullEquality>();
+ assert_datafusion_try_from::<protobuf_common::Constraint, Constraint>();
+ assert_datafusion_try_from::<&protobuf_common::Constraint, Constraint>();
+ assert_datafusion_try_from::<protobuf_common::Constraints, Constraints>();
+ assert_datafusion_try_from::<&protobuf_common::Constraints, Constraints>();
}
#[test]
fn file_format_option_conversions_are_std_traits() {
- assert_from::<&protobuf::CsvOptions, CsvOptions>();
- assert_from::<&protobuf::JsonOptions, JsonOptions>();
- assert_try_from::<&protobuf::ParquetOptions, ParquetOptions>();
+ assert_datafusion_try_from::<&protobuf::CsvOptions, CsvOptions>();
+ assert_datafusion_try_from::<&protobuf::JsonOptions, JsonOptions>();
+ assert_datafusion_try_from::<&protobuf::ParquetOptions, ParquetOptions>();
assert_from::<protobuf::ParquetColumnOptions, ParquetColumnOptions>();
- assert_from::<protobuf::ParquetCdcOptions, ParquetCdcOptions>();
- assert_try_from::<&protobuf::TableParquetOptions, TableParquetOptions>();
+ assert_datafusion_try_from::<protobuf::ParquetCdcOptions, ParquetCdcOptions>();
+ assert_datafusion_try_from::<&protobuf::TableParquetOptions, TableParquetOptions>();
assert_from::<&CsvFormatFactory, protobuf::CsvOptions>();
assert_from::<&JsonFormatFactory, protobuf::JsonOptions>();
assert_from::<&ParquetFormatFactory, protobuf::TableParquetOptions>();
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 881aee0c63..0acde5640e 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -38,6 +38,31 @@ Implement `values_preserving`, call
`selection.validate_num_groups(self.len())?` before reading, and return rows in
`selection.iter()` order without changing the builder state.
+### `datafusion-proto`: common option and constraint conversions are fallible
+
+Protobuf conversions for `CsvOptions`, `JsonOptions`, `ParquetCdcOptions`,
+`Constraint`, and `Constraints` now reject integer values that do not fit
+`usize`. Their infallible `From` implementations have been replaced with
+`TryFrom`. The existing `TryFrom` conversions for `ParquetOptions` and
+`TableParquetOptions` now also validate all `usize`-backed fields. A constraint
+without a `constraint_mode` returns an error instead of panicking.
+
+**Migration guide:**
+
+```rust,ignore
+// Before
+let csv = CsvOptions::from(&proto_csv);
+let cdc = ParquetCdcOptions::from(proto_cdc);
+let constraints: Constraints = proto_constraints.into();
+
+// After
+let csv = CsvOptions::try_from(&proto_csv)?;
+let cdc = ParquetCdcOptions::try_from(proto_cdc)?;
+let constraints = Constraints::try_from(proto_constraints)?;
+```
+
+See [issue #24170](https://github.com/apache/datafusion/issues/24170) for details.
+
### `datafusion.optimizer.use_statistics_registry` is deprecated and ignored
The `datafusion.optimizer.use_statistics_registry` config flag is deprecated and
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org