You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2024/10/06 11:20:50 UTC
(arrow-rs) branch master updated: Improve parquet `MetadataFetch` and `AsyncFileReader` docs (#6505)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 408206b588 Improve parquet `MetadataFetch` and `AsyncFileReader` docs (#6505)
408206b588 is described below
commit 408206b5883a00e7695dda030457b7ca0084be35
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Sun Oct 6 07:20:45 2024 -0400
Improve parquet `MetadataFetch` and `AsyncFileReader` docs (#6505)
* Improve MetadataFetch and AsyncFileReader docs
* Update parquet/src/arrow/async_reader/mod.rs
Co-authored-by: Matthijs Brobbel <m1...@gmail.com>
* Update parquet/src/arrow/async_reader/metadata.rs
Co-authored-by: Matthijs Brobbel <m1...@gmail.com>
* Update parquet/src/arrow/async_reader/metadata.rs
Co-authored-by: Val Lorentz <pr...@progval.net>
* Update parquet/src/file/metadata/reader.rs
Co-authored-by: Val Lorentz <pr...@progval.net>
---------
Co-authored-by: Matthijs Brobbel <m1...@gmail.com>
Co-authored-by: Val Lorentz <pr...@progval.net>
---
parquet/src/arrow/async_reader/metadata.rs | 38 +++++++++++++++++++++++++++++-
parquet/src/arrow/async_reader/mod.rs | 12 ++++++++++
parquet/src/file/metadata/reader.rs | 9 ++++++-
3 files changed, 57 insertions(+), 2 deletions(-)
diff --git a/parquet/src/arrow/async_reader/metadata.rs b/parquet/src/arrow/async_reader/metadata.rs
index b2c6159bee..afc77ae04f 100644
--- a/parquet/src/arrow/async_reader/metadata.rs
+++ b/parquet/src/arrow/async_reader/metadata.rs
@@ -28,8 +28,44 @@ use std::future::Future;
use std::ops::Range;
/// A data source that can be used with [`MetadataLoader`] to load [`ParquetMetaData`]
+///
+/// Note that implementation is provided for [`AsyncFileReader`].
+///
+/// # Example `MetadataFetch` for a custom async data source
+///
+/// ```rust
+/// # use parquet::errors::Result;
+/// # use parquet::arrow::async_reader::MetadataFetch;
+/// # use bytes::Bytes;
+/// # use std::ops::Range;
+/// # use std::io::SeekFrom;
+/// # use futures::future::BoxFuture;
+/// # use futures::FutureExt;
+/// # use tokio::io::{AsyncReadExt, AsyncSeekExt};
+/// // Adapter that implements the API for reading bytes from an async source (in
+/// // this case a tokio::fs::File)
+/// struct TokioFileMetadata {
+/// file: tokio::fs::File,
+/// }
+/// impl MetadataFetch for TokioFileMetadata {
+/// fn fetch(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes>> {
+/// // return a future that fetches data in range
+/// async move {
+/// let mut buf = vec![0; range.len()]; // target buffer
+/// // seek to the start of the range and read the data
+/// self.file.seek(SeekFrom::Start(range.start as u64)).await?;
+/// self.file.read_exact(&mut buf).await?;
+/// Ok(Bytes::from(buf)) // convert to Bytes
+/// }
+/// .boxed() // turn into BoxedFuture, using FutureExt::boxed
+/// }
+/// }
+///```
pub trait MetadataFetch {
- /// Fetches a range of bytes asynchronously
+ /// Return a future that fetches the specified range of bytes asynchronously
+ ///
+ /// Note the returned type is a boxed future, often created by
+ /// [FutureExt::boxed]. See the trait documentation for an example
fn fetch(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes>>;
}
diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs
index b521ac0102..029567d4ef 100644
--- a/parquet/src/arrow/async_reader/mod.rs
+++ b/parquet/src/arrow/async_reader/mod.rs
@@ -121,6 +121,18 @@ use crate::arrow::schema::ParquetField;
pub use store::*;
/// The asynchronous interface used by [`ParquetRecordBatchStream`] to read parquet files
+///
+/// Notes:
+///
+/// 1. There is a default implementation for types that implement [`AsyncRead`]
+/// and [`AsyncSeek`], for example [`tokio::fs::File`].
+///
+/// 2. [`ParquetObjectReader`], available when the `object_store` crate feature
+/// is enabled, implements this interface for [`ObjectStore`].
+///
+/// [`ObjectStore`]: object_store::ObjectStore
+///
+/// [`tokio::fs::File`]: https://docs.rs/tokio/latest/tokio/fs/struct.File.html
pub trait AsyncFileReader: Send {
/// Retrieve the bytes in `range`
fn get_bytes(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes>>;
diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs
index ec4b97bb57..c803d6013d 100644
--- a/parquet/src/file/metadata/reader.rs
+++ b/parquet/src/file/metadata/reader.rs
@@ -39,11 +39,18 @@ use crate::arrow::async_reader::MetadataFetch;
/// See [`crate::file::metadata::ParquetMetaDataWriter#output-format`] for a description of
/// the Parquet metadata.
///
+/// Parquet metadata is not necessarily contiguous in the files: part is stored
+/// in the footer (the last bytes of the file), but other portions (such as the
+/// PageIndex) can be stored elsewhere.
+///
+/// This reader handles reading the footer as well as the non contiguous parts
+/// of the metadata such as the page indexes; excluding Bloom Filters.
+///
/// # Example
/// ```no_run
/// # use parquet::file::metadata::ParquetMetaDataReader;
/// # fn open_parquet_file(path: &str) -> std::fs::File { unimplemented!(); }
-/// // read parquet metadata including page indexes
+/// // read parquet metadata including page indexes from a file
/// let file = open_parquet_file("some_path.parquet");
/// let mut reader = ParquetMetaDataReader::new()
/// .with_page_indexes(true);