You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2024/10/06 11:20:50 UTC

(arrow-rs) branch master updated: Improve parquet `MetadataFetch` and `AsyncFileReader` docs (#6505)

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git


The following commit(s) were added to refs/heads/master by this push:
     new 408206b588 Improve parquet `MetadataFetch` and `AsyncFileReader` docs (#6505)
408206b588 is described below

commit 408206b5883a00e7695dda030457b7ca0084be35
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Sun Oct 6 07:20:45 2024 -0400

    Improve parquet `MetadataFetch` and `AsyncFileReader` docs (#6505)
    
    * Improve MetadataFetch and AsyncFileReader docs
    
    * Update parquet/src/arrow/async_reader/mod.rs
    
    Co-authored-by: Matthijs Brobbel <m1...@gmail.com>
    
    * Update parquet/src/arrow/async_reader/metadata.rs
    
    Co-authored-by: Matthijs Brobbel <m1...@gmail.com>
    
    * Update parquet/src/arrow/async_reader/metadata.rs
    
    Co-authored-by: Val Lorentz <pr...@progval.net>
    
    * Update parquet/src/file/metadata/reader.rs
    
    Co-authored-by: Val Lorentz <pr...@progval.net>
    
    ---------
    
    Co-authored-by: Matthijs Brobbel <m1...@gmail.com>
    Co-authored-by: Val Lorentz <pr...@progval.net>
---
 parquet/src/arrow/async_reader/metadata.rs | 38 +++++++++++++++++++++++++++++-
 parquet/src/arrow/async_reader/mod.rs      | 12 ++++++++++
 parquet/src/file/metadata/reader.rs        |  9 ++++++-
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/parquet/src/arrow/async_reader/metadata.rs b/parquet/src/arrow/async_reader/metadata.rs
index b2c6159bee..afc77ae04f 100644
--- a/parquet/src/arrow/async_reader/metadata.rs
+++ b/parquet/src/arrow/async_reader/metadata.rs
@@ -28,8 +28,44 @@ use std::future::Future;
 use std::ops::Range;
 
 /// A data source that can be used with [`MetadataLoader`] to load [`ParquetMetaData`]
+///
+/// Note that implementation is provided for [`AsyncFileReader`].
+///
+/// # Example `MetadataFetch` for a custom async data source
+///
+/// ```rust
+/// # use parquet::errors::Result;
+/// # use parquet::arrow::async_reader::MetadataFetch;
+/// # use bytes::Bytes;
+/// # use std::ops::Range;
+/// # use std::io::SeekFrom;
+/// # use futures::future::BoxFuture;
+/// # use futures::FutureExt;
+/// # use tokio::io::{AsyncReadExt, AsyncSeekExt};
+/// // Adapter that implements the API for reading bytes from an async source (in
+/// // this case a tokio::fs::File)
+/// struct TokioFileMetadata {
+///     file: tokio::fs::File,
+/// }
+/// impl MetadataFetch for TokioFileMetadata {
+///     fn fetch(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes>> {
+///         // return a future that fetches data in range
+///         async move {
+///             let mut buf = vec![0; range.len()]; // target buffer
+///             // seek to the start of the range and read the data
+///             self.file.seek(SeekFrom::Start(range.start as u64)).await?;
+///             self.file.read_exact(&mut buf).await?;
+///             Ok(Bytes::from(buf)) // convert to Bytes
+///         }
+///             .boxed() // turn into BoxedFuture, using FutureExt::boxed
+///     }
+/// }
+///```
 pub trait MetadataFetch {
-    /// Fetches a range of bytes asynchronously
+    /// Return a future that fetches the specified range of bytes asynchronously
+    ///
+    /// Note the returned type is a boxed future, often created by
+    /// [FutureExt::boxed]. See the trait documentation for an example
     fn fetch(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes>>;
 }
 
diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs
index b521ac0102..029567d4ef 100644
--- a/parquet/src/arrow/async_reader/mod.rs
+++ b/parquet/src/arrow/async_reader/mod.rs
@@ -121,6 +121,18 @@ use crate::arrow::schema::ParquetField;
 pub use store::*;
 
 /// The asynchronous interface used by [`ParquetRecordBatchStream`] to read parquet files
+///
+/// Notes:
+///
+/// 1. There is a default implementation for types that implement [`AsyncRead`]
+///    and [`AsyncSeek`], for example [`tokio::fs::File`].
+///
+/// 2. [`ParquetObjectReader`], available when the `object_store` crate feature
+///    is enabled, implements this interface for [`ObjectStore`].
+///
+/// [`ObjectStore`]: object_store::ObjectStore
+///
+/// [`tokio::fs::File`]: https://docs.rs/tokio/latest/tokio/fs/struct.File.html
 pub trait AsyncFileReader: Send {
     /// Retrieve the bytes in `range`
     fn get_bytes(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes>>;
diff --git a/parquet/src/file/metadata/reader.rs b/parquet/src/file/metadata/reader.rs
index ec4b97bb57..c803d6013d 100644
--- a/parquet/src/file/metadata/reader.rs
+++ b/parquet/src/file/metadata/reader.rs
@@ -39,11 +39,18 @@ use crate::arrow::async_reader::MetadataFetch;
 /// See [`crate::file::metadata::ParquetMetaDataWriter#output-format`] for a description of
 /// the Parquet metadata.
 ///
+/// Parquet metadata is not necessarily contiguous in the files: part is stored
+/// in the footer (the last bytes of the file), but other portions (such as the
+/// PageIndex) can be stored elsewhere.
+///
+/// This reader handles reading the footer as well as the non contiguous parts
+/// of the metadata such as the page indexes; excluding Bloom Filters.
+///
 /// # Example
 /// ```no_run
 /// # use parquet::file::metadata::ParquetMetaDataReader;
 /// # fn open_parquet_file(path: &str) -> std::fs::File { unimplemented!(); }
-/// // read parquet metadata including page indexes
+/// // read parquet metadata including page indexes from a file
 /// let file = open_parquet_file("some_path.parquet");
 /// let mut reader = ParquetMetaDataReader::new()
 ///     .with_page_indexes(true);