You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/16 06:58:20 UTC
(datafusion) branch gh-readonly-queue/main/pr-23595-67907a6fc51e00296ae68d6207813d2eec57fa9b created (now 1adbd84e4f)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23595-67907a6fc51e00296ae68d6207813d2eec57fa9b
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 1adbd84e4f minor: validate config `meta_fetch_concurrency` when setting it (#23595)
This branch includes the following new commits:
new 1adbd84e4f minor: validate config `meta_fetch_concurrency` when setting it (#23595)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: minor: validate config `meta_fetch_concurrency` when setting it (#23595)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23595-67907a6fc51e00296ae68d6207813d2eec57fa9b
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 1adbd84e4f97855acf5e696945490bfe0646246a
Author: Yongting You <20...@gmail.com>
AuthorDate: Thu Jul 16 14:57:57 2026 +0800
minor: validate config `meta_fetch_concurrency` when setting it (#23595)
## Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->
- Part of https://github.com/apache/datafusion/issues/17498
## Rationale for this change
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
This is logically the same change as #23592, but for another
configuration.
0 is an invalid value for concurrency level, so restricting it to be
non-zero
## What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
## Are these changes tested?
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
-->
## Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->
<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->
---
datafusion/catalog-listing/src/table.rs | 6 ++-
datafusion/common/src/config.rs | 8 ++--
datafusion/core/src/datasource/listing/table.rs | 3 +-
datafusion/core/tests/config_from_env.rs | 5 ++-
datafusion/datasource-parquet/src/file_format.rs | 8 +++-
datafusion/sqllogictest/test_files/explain.slt | 7 +++-
.../sqllogictest/test_files/set_variable.slt | 43 +++++++++++++++++-----
.../test_files/spark/map/str_to_map.slt | 6 ++-
8 files changed, 65 insertions(+), 21 deletions(-)
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs
index 23c67efa74..ce739e5019 100644
--- a/datafusion/catalog-listing/src/table.rs
+++ b/datafusion/catalog-listing/src/table.rs
@@ -815,7 +815,7 @@ impl ListingTable {
}))
.await?;
let meta_fetch_concurrency =
- ctx.config_options().execution.meta_fetch_concurrency;
+ ctx.config_options().execution.meta_fetch_concurrency.get();
let file_list = stream::iter(file_list).flatten_unordered(meta_fetch_concurrency);
// collect the statistics and ordering if required by the config
let files = file_list
@@ -832,7 +832,9 @@ impl ListingTable {
.with_ordering(ordering))
})
.boxed()
- .buffer_unordered(ctx.config_options().execution.meta_fetch_concurrency);
+ .buffer_unordered(
+ ctx.config_options().execution.meta_fetch_concurrency.get(),
+ );
get_files_with_limit(files, file_limit, ctx.config().collect_statistics()).await
}
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index bb2e36fc5f..81f573fc2a 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -861,7 +861,7 @@ config_namespace! {
pub max_spill_file_size_bytes: ConfigNonZeroUsize, default = non_zero_usize_default(128 * 1024 * 1024)
/// Number of files to read in parallel when inferring schema and statistics
- pub meta_fetch_concurrency: usize, default = 32
+ pub meta_fetch_concurrency: ConfigNonZeroUsize, default = non_zero_usize_default(32)
/// Guarantees a minimum level of output files running in parallel.
/// RecordBatches will be distributed in round robin fashion to each
@@ -1932,7 +1932,8 @@ impl ConfigOptions {
}
return Ok(());
}
- return ConfigField::set(self, inner_key, value);
+ return ConfigField::set(self, inner_key, value)
+ .map_err(|e| e.context(format!("Error setting config {key}")));
}
if !self.extensions.0.contains_key(prefix)
@@ -3771,6 +3772,7 @@ impl Display for OutputFormat {
#[cfg(test)]
mod tests {
#[cfg(feature = "parquet")]
+ use crate::assert_contains;
use crate::config::TableParquetOptions;
use crate::config::{
ConfigEntry, ConfigExtension, ConfigField, ConfigFileType, ExtensionOptions,
@@ -4331,7 +4333,7 @@ mod tests {
let err = config
.set("datafusion.execution.parquet.writer_version", "3.0")
.unwrap_err();
- assert_eq!(
+ assert_contains!(
err.to_string(),
"Invalid or Unsupported Configuration: Invalid parquet writer version: 3.0. Expected one of: 1.0, 2.0"
);
diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs
index db2623c24d..56a5d57795 100644
--- a/datafusion/core/src/datasource/listing/table.rs
+++ b/datafusion/core/src/datasource/listing/table.rs
@@ -542,7 +542,8 @@ mod tests {
.state()
.config_options()
.execution
- .meta_fetch_concurrency;
+ .meta_fetch_concurrency
+ .get();
let expected_concurrency = files.len().min(meta_fetch_concurrency);
let head_concurrency_store = ensure_head_concurrency(store, expected_concurrency);
diff --git a/datafusion/core/tests/config_from_env.rs b/datafusion/core/tests/config_from_env.rs
index 6b09a6367d..15a047cbbd 100644
--- a/datafusion/core/tests/config_from_env.rs
+++ b/datafusion/core/tests/config_from_env.rs
@@ -16,6 +16,7 @@
// under the License.
use datafusion::config::ConfigOptions;
+use datafusion_common::assert_contains;
use std::env;
#[test]
@@ -34,7 +35,7 @@ fn from_env() {
// invalid testing
env::set_var(env_key, "ttruee");
let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();
- assert_eq!(
+ assert_contains!(
err,
"Error parsing 'ttruee' as bool\ncaused by\nExternal error: provided string was not `true` or `false`"
);
@@ -50,7 +51,7 @@ fn from_env() {
// for invalid testing
env::set_var(env_key, "abc");
let err = ConfigOptions::from_env().unwrap_err().strip_backtrace();
- assert_eq!(
+ assert_contains!(
err,
"Error parsing 'abc' as usize\ncaused by\nExternal error: invalid digit found in string"
);
diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs
index e89cff2aaf..29083ebfb2 100644
--- a/datafusion/datasource-parquet/src/file_format.rs
+++ b/datafusion/datasource-parquet/src/file_format.rs
@@ -367,7 +367,13 @@ impl FileFormat for ParquetFormat {
})
.boxed() // Workaround https://github.com/rust-lang/rust/issues/64552
// fetch schemas concurrently, if requested
- .buffer_unordered(state.config_options().execution.meta_fetch_concurrency)
+ .buffer_unordered(
+ state
+ .config_options()
+ .execution
+ .meta_fetch_concurrency
+ .get(),
+ )
.try_collect()
.await?;
diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt
index 24b1262e02..5405c7ce0e 100644
--- a/datafusion/sqllogictest/test_files/explain.slt
+++ b/datafusion/sqllogictest/test_files/explain.slt
@@ -533,8 +533,13 @@ query error DataFusion error: Error during planning: EXPLAIN VERBOSE with FORMAT
explain verbose format tree select * from values (1);
# valid explain format
-query error DataFusion error: Invalid or Unsupported Configuration: Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got 'xxx'
+query error
set datafusion.explain.format = "xxx";
+----
+DataFusion error: Error setting config datafusion.explain.format
+caused by
+Invalid or Unsupported Configuration: Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got 'xxx'
+
# verbose uses indent mode even when a different mode (e.g tree) is set
diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.slt
index 15267e6a2a..7da06b2fff 100644
--- a/datafusion/sqllogictest/test_files/set_variable.slt
+++ b/datafusion/sqllogictest/test_files/set_variable.slt
@@ -93,10 +93,10 @@ datafusion.execution.coalesce_batches false
statement ok
set datafusion.catalog.information_schema = true
-statement error DataFusion error: Error parsing '1' as bool
+statement error DataFusion error: Error setting config datafusion\.execution\.coalesce_batches\ncaused by\nError parsing '1' as bool
SET datafusion.execution.coalesce_batches to 1
-statement error DataFusion error: Error parsing 'abc' as bool
+statement error DataFusion error: Error setting config datafusion\.execution\.coalesce_batches\ncaused by\nError parsing 'abc' as bool
SET datafusion.execution.coalesce_batches to abc
# set u64 variable
@@ -132,10 +132,10 @@ datafusion.execution.batch_size 2
statement ok
set datafusion.catalog.information_schema = true
-statement error DataFusion error: Error parsing '-1' as usize
+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nError parsing '-1' as usize
SET datafusion.execution.batch_size to -1
-statement error DataFusion error: Error parsing 'abc' as usize
+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nError parsing 'abc' as usize
SET datafusion.execution.batch_size to abc
statement error External error: invalid digit found in string
@@ -580,7 +580,7 @@ SHOW datafusion.format.date_format
datafusion.format.date_format %Y-%m-%d
# Invalid format option name
-statement error DataFusion error: Invalid or Unsupported Configuration: Config value "unknown_option" not found on FormatOptions
+statement error DataFusion error: Error setting config datafusion\.format\.unknown_option\ncaused by\nInvalid or Unsupported Configuration: Config value "unknown_option" not found on FormatOptions
SET datafusion.format.unknown_option = true
############
@@ -721,20 +721,43 @@ statement error DataFusion error: Error during planning: Duration has overflowed
SET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'
# Set invalid value and ensures error
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error DataFusion error: Error setting config datafusion\.execution\.batch_size\ncaused by\nInvalid or Unsupported Configuration: value must be greater than 0
SET datafusion.execution.batch_size = 0
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error DataFusion error: Error setting config datafusion\.execution\.meta_fetch_concurrency\ncaused by\nInvalid or Unsupported Configuration: value must be greater than 0
+SET datafusion.execution.meta_fetch_concurrency = 0
+
+statement error
SET datafusion.execution.minimum_parallel_output_files = 0
+----
+DataFusion error: Error setting config datafusion.execution.minimum_parallel_output_files
+caused by
+Invalid or Unsupported Configuration: value must be greater than 0
+
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error
SET datafusion.execution.soft_max_rows_per_output_file = 0
+----
+DataFusion error: Error setting config datafusion.execution.soft_max_rows_per_output_file
+caused by
+Invalid or Unsupported Configuration: value must be greater than 0
+
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error
SET datafusion.execution.max_spill_file_size_bytes = 0
+----
+DataFusion error: Error setting config datafusion.execution.max_spill_file_size_bytes
+caused by
+Invalid or Unsupported Configuration: value must be greater than 0
+
-statement error DataFusion error: Invalid or Unsupported Configuration: value must be greater than 0
+statement error
SET datafusion.sql_parser.recursion_limit = 0
+----
+DataFusion error: Error setting config datafusion.sql_parser.recursion_limit
+caused by
+Invalid or Unsupported Configuration: value must be greater than 0
+
# Config reset
statement ok
diff --git a/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt b/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt
index f422b50dfa..c1307468d7 100644
--- a/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt
+++ b/datafusion/sqllogictest/test_files/spark/map/str_to_map.slt
@@ -159,5 +159,9 @@ statement ok
set datafusion.spark.map_key_dedup_policy = 'EXCEPTION';
# Invalid policy values are rejected at SET time with a clear message.
-statement error DataFusion error: Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS\. Expected one of: EXCEPTION, LAST_WIN
+statement error
set datafusion.spark.map_key_dedup_policy = 'BOGUS';
+----
+DataFusion error: Error setting config datafusion.spark.map_key_dedup_policy
+caused by
+Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org