You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by al...@apache.org on 2024/07/17 19:08:51 UTC
(datafusion) branch main updated: Refactor: more clearly delineate between `TableParquetOptions` and `ParquetWriterOptions` (#11444)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new c95556def3 Refactor: more clearly delineate between `TableParquetOptions` and `ParquetWriterOptions` (#11444)
c95556def3 is described below
commit c95556def34bab6a7b4a74467eb08a92f67426af
Author: wiedld <wi...@users.noreply.github.com>
AuthorDate: Wed Jul 17 12:08:48 2024 -0700
Refactor: more clearly delineate between `TableParquetOptions` and `ParquetWriterOptions` (#11444)
* refactor: make more explicit the relationship btwn TableParquetOptions vs ParquetOptions vs WriterProperties
* test: demonstrate the relationship btwn session configs and writer props
* refactor: move parquet-format specific functionality to the parquet submodule, leaving only the config options in the config module.
* test: update test fixtures to use the ParquetOptions::default
* test: update test helper session_config_from_writer_props, to not add column configuration when none exists
* test(11367): write test to demonstrate issue 11367
* fix: existing sqllogictests require specific ParquetOptions settings to be left as None
* test(11367): demonstrate how the require bloom filter defaults, (required to avoid test regression), result in different default behavior than parquet crate
* chore: make more reviewable, by pulling tests for issue 11367 into followup PR
* refactor: move all parquet-associated features into parquet-writer mod
* chore: better function naming convention
---
.../common/src/file_options/parquet_writer.rs | 409 ++++++++++++++++-----
1 file changed, 320 insertions(+), 89 deletions(-)
diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rs
index 009164a29e..a14cbdecf6 100644
--- a/datafusion/common/src/file_options/parquet_writer.rs
+++ b/datafusion/common/src/file_options/parquet_writer.rs
@@ -24,10 +24,11 @@ use crate::{
use parquet::{
basic::{BrotliLevel, GzipLevel, ZstdLevel},
- file::{
- metadata::KeyValue,
- properties::{EnabledStatistics, WriterProperties, WriterVersion},
+ file::properties::{
+ EnabledStatistics, WriterProperties, WriterPropertiesBuilder, WriterVersion,
+ DEFAULT_MAX_STATISTICS_SIZE, DEFAULT_STATISTICS_ENABLED,
},
+ format::KeyValue,
schema::types::ColumnPath,
};
@@ -52,92 +53,43 @@ impl ParquetWriterOptions {
impl TryFrom<&TableParquetOptions> for ParquetWriterOptions {
type Error = DataFusionError;
- fn try_from(parquet_options: &TableParquetOptions) -> Result<Self> {
- let ParquetOptions {
- data_pagesize_limit,
- write_batch_size,
- writer_version,
- dictionary_page_size_limit,
- max_row_group_size,
- created_by,
- column_index_truncate_length,
- data_page_row_count_limit,
- bloom_filter_on_write,
- encoding,
- dictionary_enabled,
- compression,
- statistics_enabled,
- max_statistics_size,
- bloom_filter_fpp,
- bloom_filter_ndv,
- // below is not part of ParquetWriterOptions
- enable_page_index: _,
- pruning: _,
- skip_metadata: _,
- metadata_size_hint: _,
- pushdown_filters: _,
- reorder_filters: _,
- allow_single_file_parallelism: _,
- maximum_parallel_row_group_writers: _,
- maximum_buffered_record_batches_per_stream: _,
- bloom_filter_on_read: _,
- } = &parquet_options.global;
-
- let key_value_metadata = if !parquet_options.key_value_metadata.is_empty() {
- Some(
- parquet_options
- .key_value_metadata
- .clone()
- .drain()
- .map(|(key, value)| KeyValue { key, value })
- .collect::<Vec<_>>(),
- )
- } else {
- None
- };
-
- let mut builder = WriterProperties::builder()
- .set_data_page_size_limit(*data_pagesize_limit)
- .set_write_batch_size(*write_batch_size)
- .set_writer_version(parse_version_string(writer_version.as_str())?)
- .set_dictionary_page_size_limit(*dictionary_page_size_limit)
- .set_max_row_group_size(*max_row_group_size)
- .set_created_by(created_by.clone())
- .set_column_index_truncate_length(*column_index_truncate_length)
- .set_data_page_row_count_limit(*data_page_row_count_limit)
- .set_bloom_filter_enabled(*bloom_filter_on_write)
- .set_key_value_metadata(key_value_metadata);
-
- if let Some(encoding) = &encoding {
- builder = builder.set_encoding(parse_encoding_string(encoding)?);
- }
-
- if let Some(enabled) = dictionary_enabled {
- builder = builder.set_dictionary_enabled(*enabled);
- }
-
- if let Some(compression) = &compression {
- builder = builder.set_compression(parse_compression_string(compression)?);
- }
-
- if let Some(statistics) = &statistics_enabled {
- builder =
- builder.set_statistics_enabled(parse_statistics_string(statistics)?);
- }
-
- if let Some(size) = max_statistics_size {
- builder = builder.set_max_statistics_size(*size);
- }
+ fn try_from(parquet_table_options: &TableParquetOptions) -> Result<Self> {
+ // ParquetWriterOptions will have defaults for the remaining fields (e.g. sorting_columns)
+ Ok(ParquetWriterOptions {
+ writer_options: WriterPropertiesBuilder::try_from(parquet_table_options)?
+ .build(),
+ })
+ }
+}
- if let Some(fpp) = bloom_filter_fpp {
- builder = builder.set_bloom_filter_fpp(*fpp);
- }
+impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder {
+ type Error = DataFusionError;
- if let Some(ndv) = bloom_filter_ndv {
- builder = builder.set_bloom_filter_ndv(*ndv);
+ /// Convert the session's [`TableParquetOptions`] into a single write action's [`WriterPropertiesBuilder`].
+ ///
+ /// The returned [`WriterPropertiesBuilder`] includes customizations applicable per column.
+ fn try_from(table_parquet_options: &TableParquetOptions) -> Result<Self> {
+ // Table options include kv_metadata and col-specific options
+ let TableParquetOptions {
+ global,
+ column_specific_options,
+ key_value_metadata,
+ } = table_parquet_options;
+
+ let mut builder = global.into_writer_properties_builder()?;
+
+ if !key_value_metadata.is_empty() {
+ builder = builder.set_key_value_metadata(Some(
+ key_value_metadata
+ .to_owned()
+ .drain()
+ .map(|(key, value)| KeyValue { key, value })
+ .collect(),
+ ));
}
- for (column, options) in &parquet_options.column_specific_options {
+ // Apply column-specific options:
+ for (column, options) in column_specific_options {
let path = ColumnPath::new(column.split('.').map(|s| s.to_owned()).collect());
if let Some(bloom_filter_enabled) = options.bloom_filter_enabled {
@@ -183,10 +135,87 @@ impl TryFrom<&TableParquetOptions> for ParquetWriterOptions {
}
}
- // ParquetWriterOptions will have defaults for the remaining fields (e.g. sorting_columns)
- Ok(ParquetWriterOptions {
- writer_options: builder.build(),
- })
+ Ok(builder)
+ }
+}
+
+impl ParquetOptions {
+ /// Convert the global session options, [`ParquetOptions`], into a single write action's [`WriterPropertiesBuilder`].
+ ///
+ /// The returned [`WriterPropertiesBuilder`] can then be further modified with additional options
+ /// applied per column; a customization which is not applicable for [`ParquetOptions`].
+ pub fn into_writer_properties_builder(&self) -> Result<WriterPropertiesBuilder> {
+ let ParquetOptions {
+ data_pagesize_limit,
+ write_batch_size,
+ writer_version,
+ compression,
+ dictionary_enabled,
+ dictionary_page_size_limit,
+ statistics_enabled,
+ max_statistics_size,
+ max_row_group_size,
+ created_by,
+ column_index_truncate_length,
+ data_page_row_count_limit,
+ encoding,
+ bloom_filter_on_write,
+ bloom_filter_fpp,
+ bloom_filter_ndv,
+
+ // not in WriterProperties
+ enable_page_index: _,
+ pruning: _,
+ skip_metadata: _,
+ metadata_size_hint: _,
+ pushdown_filters: _,
+ reorder_filters: _,
+ allow_single_file_parallelism: _,
+ maximum_parallel_row_group_writers: _,
+ maximum_buffered_record_batches_per_stream: _,
+ bloom_filter_on_read: _, // reads not used for writer props
+ } = self;
+
+ let mut builder = WriterProperties::builder()
+ .set_data_page_size_limit(*data_pagesize_limit)
+ .set_write_batch_size(*write_batch_size)
+ .set_writer_version(parse_version_string(writer_version.as_str())?)
+ .set_dictionary_page_size_limit(*dictionary_page_size_limit)
+ .set_statistics_enabled(
+ statistics_enabled
+ .as_ref()
+ .and_then(|s| parse_statistics_string(s).ok())
+ .unwrap_or(DEFAULT_STATISTICS_ENABLED),
+ )
+ .set_max_statistics_size(
+ max_statistics_size.unwrap_or(DEFAULT_MAX_STATISTICS_SIZE),
+ )
+ .set_max_row_group_size(*max_row_group_size)
+ .set_created_by(created_by.clone())
+ .set_column_index_truncate_length(*column_index_truncate_length)
+ .set_data_page_row_count_limit(*data_page_row_count_limit)
+ .set_bloom_filter_enabled(*bloom_filter_on_write);
+
+ if let Some(bloom_filter_fpp) = bloom_filter_fpp {
+ builder = builder.set_bloom_filter_fpp(*bloom_filter_fpp);
+ };
+ if let Some(bloom_filter_ndv) = bloom_filter_ndv {
+ builder = builder.set_bloom_filter_ndv(*bloom_filter_ndv);
+ };
+ if let Some(dictionary_enabled) = dictionary_enabled {
+ builder = builder.set_dictionary_enabled(*dictionary_enabled);
+ };
+
+ // We do not have access to default ColumnProperties set in Arrow.
+ // Therefore, only overwrite if these settings exist.
+ if let Some(compression) = compression {
+ builder = builder.set_compression(parse_compression_string(compression)?);
+ }
+ if let Some(encoding) = encoding {
+ builder = builder.set_encoding(parse_encoding_string(encoding)?);
+ }
+
+ Ok(builder)
}
}
@@ -336,3 +365,205 @@ pub(crate) fn parse_statistics_string(str_setting: &str) -> Result<EnabledStatis
))),
}
}
+
+#[cfg(feature = "parquet")]
+#[cfg(test)]
+mod tests {
+ use parquet::{basic::Compression, file::properties::EnabledStatistics};
+ use std::collections::HashMap;
+
+ use crate::config::{ColumnOptions, ParquetOptions};
+
+ use super::*;
+
+ const COL_NAME: &str = "configured";
+
+ /// Take the column defaults provided in [`ParquetOptions`], and generate a non-default col config.
+ fn column_options_with_non_defaults(
+ src_col_defaults: &ParquetOptions,
+ ) -> ColumnOptions {
+ ColumnOptions {
+ compression: Some("zstd(22)".into()),
+ dictionary_enabled: src_col_defaults.dictionary_enabled.map(|v| !v),
+ statistics_enabled: Some("page".into()),
+ max_statistics_size: Some(72),
+ encoding: Some("RLE".into()),
+ bloom_filter_enabled: Some(true),
+ bloom_filter_fpp: Some(0.72),
+ bloom_filter_ndv: Some(72),
+ }
+ }
+
+ fn parquet_options_with_non_defaults() -> ParquetOptions {
+ let defaults = ParquetOptions::default();
+ let writer_version = if defaults.writer_version.eq("1.0") {
+ "2.0"
+ } else {
+ "1.0"
+ };
+
+ ParquetOptions {
+ data_pagesize_limit: 42,
+ write_batch_size: 42,
+ writer_version: writer_version.into(),
+ compression: Some("zstd(22)".into()),
+ dictionary_enabled: Some(!defaults.dictionary_enabled.unwrap_or(false)),
+ dictionary_page_size_limit: 42,
+ statistics_enabled: Some("chunk".into()),
+ max_statistics_size: Some(42),
+ max_row_group_size: 42,
+ created_by: "wordy".into(),
+ column_index_truncate_length: Some(42),
+ data_page_row_count_limit: 42,
+ encoding: Some("BYTE_STREAM_SPLIT".into()),
+ bloom_filter_on_write: !defaults.bloom_filter_on_write,
+ bloom_filter_fpp: Some(0.42),
+ bloom_filter_ndv: Some(42),
+
+ // not in WriterProperties, but itemizing here to not skip newly added props
+ enable_page_index: defaults.enable_page_index,
+ pruning: defaults.pruning,
+ skip_metadata: defaults.skip_metadata,
+ metadata_size_hint: defaults.metadata_size_hint,
+ pushdown_filters: defaults.pushdown_filters,
+ reorder_filters: defaults.reorder_filters,
+ allow_single_file_parallelism: defaults.allow_single_file_parallelism,
+ maximum_parallel_row_group_writers: defaults
+ .maximum_parallel_row_group_writers,
+ maximum_buffered_record_batches_per_stream: defaults
+ .maximum_buffered_record_batches_per_stream,
+ bloom_filter_on_read: defaults.bloom_filter_on_read,
+ }
+ }
+
+ fn extract_column_options(
+ props: &WriterProperties,
+ col: ColumnPath,
+ ) -> ColumnOptions {
+ let bloom_filter_default_props = props.bloom_filter_properties(&col);
+
+ ColumnOptions {
+ bloom_filter_enabled: Some(bloom_filter_default_props.is_some()),
+ encoding: props.encoding(&col).map(|s| s.to_string()),
+ dictionary_enabled: Some(props.dictionary_enabled(&col)),
+ compression: match props.compression(&col) {
+ Compression::ZSTD(lvl) => {
+ Some(format!("zstd({})", lvl.compression_level()))
+ }
+ _ => None,
+ },
+ statistics_enabled: Some(
+ match props.statistics_enabled(&col) {
+ EnabledStatistics::None => "none",
+ EnabledStatistics::Chunk => "chunk",
+ EnabledStatistics::Page => "page",
+ }
+ .into(),
+ ),
+ bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp),
+ bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv),
+ max_statistics_size: Some(props.max_statistics_size(&col)),
+ }
+ }
+
+ /// For testing only, take a single write's props and convert back into the session config.
+ /// (use identity to confirm correct.)
+ fn session_config_from_writer_props(props: &WriterProperties) -> TableParquetOptions {
+ let default_col = ColumnPath::from("col doesn't have specific config");
+ let default_col_props = extract_column_options(props, default_col);
+
+ let configured_col = ColumnPath::from(COL_NAME);
+ let configured_col_props = extract_column_options(props, configured_col);
+
+ let key_value_metadata = props
+ .key_value_metadata()
+ .map(|pairs| {
+ HashMap::from_iter(
+ pairs
+ .iter()
+ .cloned()
+ .map(|KeyValue { key, value }| (key, value)),
+ )
+ })
+ .unwrap_or_default();
+
+ let global_options_defaults = ParquetOptions::default();
+
+ let column_specific_options = if configured_col_props.eq(&default_col_props) {
+ HashMap::default()
+ } else {
+ HashMap::from([(COL_NAME.into(), configured_col_props)])
+ };
+
+ TableParquetOptions {
+ global: ParquetOptions {
+ // global options
+ data_pagesize_limit: props.dictionary_page_size_limit(),
+ write_batch_size: props.write_batch_size(),
+ writer_version: format!("{}.0", props.writer_version().as_num()),
+ dictionary_page_size_limit: props.dictionary_page_size_limit(),
+ max_row_group_size: props.max_row_group_size(),
+ created_by: props.created_by().to_string(),
+ column_index_truncate_length: props.column_index_truncate_length(),
+ data_page_row_count_limit: props.data_page_row_count_limit(),
+
+ // global options which set the default column props
+ encoding: default_col_props.encoding,
+ compression: default_col_props.compression,
+ dictionary_enabled: default_col_props.dictionary_enabled,
+ statistics_enabled: default_col_props.statistics_enabled,
+ max_statistics_size: default_col_props.max_statistics_size,
+ bloom_filter_on_write: default_col_props
+ .bloom_filter_enabled
+ .unwrap_or_default(),
+ bloom_filter_fpp: default_col_props.bloom_filter_fpp,
+ bloom_filter_ndv: default_col_props.bloom_filter_ndv,
+
+ // not in WriterProperties
+ enable_page_index: global_options_defaults.enable_page_index,
+ pruning: global_options_defaults.pruning,
+ skip_metadata: global_options_defaults.skip_metadata,
+ metadata_size_hint: global_options_defaults.metadata_size_hint,
+ pushdown_filters: global_options_defaults.pushdown_filters,
+ reorder_filters: global_options_defaults.reorder_filters,
+ allow_single_file_parallelism: global_options_defaults
+ .allow_single_file_parallelism,
+ maximum_parallel_row_group_writers: global_options_defaults
+ .maximum_parallel_row_group_writers,
+ maximum_buffered_record_batches_per_stream: global_options_defaults
+ .maximum_buffered_record_batches_per_stream,
+ bloom_filter_on_read: global_options_defaults.bloom_filter_on_read,
+ },
+ column_specific_options,
+ key_value_metadata,
+ }
+ }
+
+ #[test]
+ fn table_parquet_opts_to_writer_props() {
+ // ParquetOptions, all props set to non-default
+ let parquet_options = parquet_options_with_non_defaults();
+
+ // TableParquetOptions, using ParquetOptions for global settings
+ let key = "foo".to_string();
+ let value = Some("bar".into());
+ let table_parquet_opts = TableParquetOptions {
+ global: parquet_options.clone(),
+ column_specific_options: [(
+ COL_NAME.into(),
+ column_options_with_non_defaults(&parquet_options),
+ )]
+ .into(),
+ key_value_metadata: [(key.clone(), value.clone())].into(),
+ };
+
+ let writer_props = WriterPropertiesBuilder::try_from(&table_parquet_opts)
+ .unwrap()
+ .build();
+ assert_eq!(
+ table_parquet_opts,
+ session_config_from_writer_props(&writer_props),
+ "the writer_props should have the same configuration as the session's TableParquetOptions",
+ );
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org