You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/01 17:55:07 UTC
(datafusion) branch gh-readonly-queue/main/pr-23253-e3fcd5808e0e55a6b12bd3b0b68e89ff0617cbd1 created (now 6583808429)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23253-e3fcd5808e0e55a6b12bd3b0b68e89ff0617cbd1
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 6583808429 feat: Expose cache hits in statistics_cache function (#23253)
This branch includes the following new commits:
new 6583808429 feat: Expose cache hits in statistics_cache function (#23253)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: feat: Expose cache hits in statistics_cache function (#23253)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23253-e3fcd5808e0e55a6b12bd3b0b68e89ff0617cbd1
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 6583808429676b6563f0a62407b68246f8f61e29
Author: Michael Kleen <mk...@gmail.com>
AuthorDate: Wed Jul 1 19:54:43 2026 +0200
feat: Expose cache hits in statistics_cache function (#23253)
## Which issue does this PR close?
- Closes None.
## Rationale for this change
Follow up to https://github.com/apache/datafusion/pull/22613. Cache hits
are now supported for all memory-limiting caches. Therefore it makes
sense to expose them in the `statistics_cache` function the same way the
`metadata_cache` function does it already.
## What changes are included in this PR?
- Add cache hits to the `statistics_cache` function
- Tests
- Adapt documentation for the `statistics_cache` function
## Are these changes tested?
Yes.
## Are there any user-facing changes?
Yes, the `statistics_cache` function supports now cache hits but no
breaking changes.
---
datafusion-cli/src/functions.rs | 4 +++
datafusion-cli/src/main.rs | 49 +++++++++++++++++++++++++++------
docs/source/user-guide/cli/functions.md | 16 ++++++-----
3 files changed, 53 insertions(+), 16 deletions(-)
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rs
index cb2372958e..7d87e7ed8a 100644
--- a/datafusion-cli/src/functions.rs
+++ b/datafusion-cli/src/functions.rs
@@ -649,6 +649,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
Field::new("num_columns", DataType::UInt64, false),
Field::new("table_size_bytes", DataType::Utf8, false),
Field::new("statistics_size_bytes", DataType::UInt64, false),
+ Field::new("hits", DataType::UInt64, false),
]));
// construct record batch from metadata
@@ -662,6 +663,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
let mut num_columns_arr = vec![];
let mut table_size_bytes_arr = vec![];
let mut statistics_size_bytes_arr = vec![];
+ let mut hits_arr = vec![];
if let Some(file_statistics_cache) = self.cache_manager.get_file_statistic_cache()
{
@@ -686,6 +688,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
.heap_size(&mut DFHeapSizeCtx::default())
as u64,
);
+ hits_arr.push(entry.hits as u64);
}
}
@@ -702,6 +705,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
Arc::new(UInt64Array::from(num_columns_arr)),
Arc::new(StringArray::from(table_size_bytes_arr)),
Arc::new(UInt64Array::from(statistics_size_bytes_arr)),
+ Arc::new(UInt64Array::from(hits_arr)),
],
)?;
diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs
index cf77e5415d..78d8342020 100644
--- a/datafusion-cli/src/main.rs
+++ b/datafusion-cli/src/main.rs
@@ -711,17 +711,48 @@ mod tests {
.await?;
}
- let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, table_size_bytes from statistics_cache() order by filename";
+ let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, hits, table_size_bytes from statistics_cache() order by filename";
let df = ctx.sql(sql).await?;
let rbs = df.collect().await?;
- assert_snapshot!(batches_to_string(&rbs),@r"
- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
- | filename | table | file_size_bytes | num_rows | num_columns | table_size_bytes |
- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
- | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | Absent |
- | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | Absent |
- | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | Absent |
- +-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
+ assert_snapshot!(batches_to_string(&rbs),@"
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ | filename | table | file_size_bytes | num_rows | num_columns | hits | table_size_bytes |
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | 0 | Absent |
+ | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | 0 | Absent |
+ | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | 0 | Absent |
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ ");
+
+ // increase the number of hits
+ ctx.sql("select * from alltypes_plain")
+ .await?
+ .collect()
+ .await?;
+ ctx.sql("select * from alltypes_plain")
+ .await?
+ .collect()
+ .await?;
+ ctx.sql("select * from alltypes_plain")
+ .await?
+ .collect()
+ .await?;
+ ctx.sql("select * from lz4_raw_compressed_larger")
+ .await?
+ .collect()
+ .await?;
+
+ let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, hits, table_size_bytes from statistics_cache() order by filename";
+ let df = ctx.sql(sql).await?;
+ let rbs = df.collect().await?;
+ assert_snapshot!(batches_to_string(&rbs),@"
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ | filename | table | file_size_bytes | num_rows | num_columns | hits | table_size_bytes |
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
+ | alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | 3 | Absent |
+ | alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | 0 | Absent |
+ | lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | 1 | Absent |
+ +-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
");
Ok(())
diff --git a/docs/source/user-guide/cli/functions.md b/docs/source/user-guide/cli/functions.md
index ea353d5c8d..409661ac82 100644
--- a/docs/source/user-guide/cli/functions.md
+++ b/docs/source/user-guide/cli/functions.md
@@ -168,6 +168,7 @@ The columns of the returned table are:
| num_rows | Utf8 | Number of rows in the table |
| num_columns | UInt64 | Number of columns in the table |
| table_size_bytes | Utf8 | Size of the table, in bytes |
+| hits | UInt64 | Number of times the cached file statistics has been accessed |
| statistics_size_bytes | UInt64 | Size of the cached statistics in memory |
## `list_files_cache`
@@ -200,13 +201,14 @@ location 's3://overturemaps-us-west-2/release/2025-12-17.0/theme=base/type=infra
```
The columns of the returned table are:
-| column_name | data_type | Description |
-| ------------------- | ------------ | ----------------------------------------------------------------------------------------- |
-| table | Utf8 | Name of the table |
-| path | Utf8 | File path relative to the object store / filesystem root |
-| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |
-| expires_in | Duration(ms) | Last modified time of the file |
-| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |
+
+| column_name | data_type | Description |
+| ------------------- | ------------ | ------------------------------------------------------------------- |
+| table | Utf8 | Name of the table |
+| path | Utf8 | File path relative to the object store / filesystem root |
+| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |
+| expires_in | Duration(ms) | Last modified time of the file |
+| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |
A metadata struct in the metadata_list contains the following fields:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org