You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2024/10/30 07:25:59 UTC
(arrow-rs) branch master updated: Parquet AsyncReader: Don't panic when empty offset_index is Some([]) (#6582)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 56d471388b Parquet AsyncReader: Don't panic when empty offset_index is Some([]) (#6582)
56d471388b is described below
commit 56d471388bae2ea5462b0d67bb93a51efbb2bce7
Author: jroddev <jr...@gmail.com>
AuthorDate: Wed Oct 30 18:25:54 2024 +1100
Parquet AsyncReader: Don't panic when empty offset_index is Some([]) (#6582)
* Don't panic when empty offset_index is Some([])
* Remove duplicate test
* Add a test with offset_index where page_locations doesnt end up as None
* remove unused imports
* Update parquet/src/arrow/async_reader/mod.rs
Co-authored-by: Ed Seidl <et...@users.noreply.github.com>
* Rename test, add explanatory comments
* cargo fmt
---------
Co-authored-by: jroddev <jr...@jroddev.com>
Co-authored-by: Ed Seidl <et...@users.noreply.github.com>
Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
---
parquet/src/arrow/async_reader/mod.rs | 105 ++++++++++++++++++++++++++++++++++
1 file changed, 105 insertions(+)
diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs
index 1c8b90b9b8..8b315cc9f7 100644
--- a/parquet/src/arrow/async_reader/mod.rs
+++ b/parquet/src/arrow/async_reader/mod.rs
@@ -506,6 +506,8 @@ where
let offset_index = self
.metadata
.offset_index()
+ // filter out empty offset indexes (old versions specified Some(vec![]) when no present)
+ .filter(|index| !index.is_empty())
.map(|x| x[row_group_idx].as_slice());
let mut row_group = InMemoryRowGroup {
@@ -828,6 +830,8 @@ impl RowGroups for InMemoryRowGroup<'_> {
Some(data) => {
let page_locations = self
.offset_index
+ // filter out empty offset indexes (old versions specified Some(vec![]) when no present)
+ .filter(|index| !index.is_empty())
.map(|index| index[i].page_locations.clone());
let page_reader: Box<dyn PageReader> = Box::new(SerializedPageReader::new(
data.clone(),
@@ -2037,4 +2041,105 @@ mod tests {
// Should only have made 3 requests
assert_eq!(requests.lock().unwrap().len(), 3);
}
+
+ #[tokio::test]
+ async fn empty_offset_index_doesnt_panic_in_read_row_group() {
+ use tokio::fs::File;
+ let testdata = arrow::util::test_util::parquet_test_data();
+ let path = format!("{testdata}/alltypes_plain.parquet");
+ let mut file = File::open(&path).await.unwrap();
+ let file_size = file.metadata().await.unwrap().len();
+ let mut metadata = ParquetMetaDataReader::new()
+ .with_page_indexes(true)
+ .load_and_finish(&mut file, file_size as usize)
+ .await
+ .unwrap();
+
+ metadata.set_offset_index(Some(vec![]));
+ let options = ArrowReaderOptions::new().with_page_index(true);
+ let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap();
+ let reader =
+ ParquetRecordBatchStreamBuilder::new_with_metadata(file, arrow_reader_metadata)
+ .build()
+ .unwrap();
+
+ let result = reader.try_collect::<Vec<_>>().await.unwrap();
+ assert_eq!(result.len(), 1);
+ }
+
+ #[tokio::test]
+ async fn non_empty_offset_index_doesnt_panic_in_read_row_group() {
+ use tokio::fs::File;
+ let testdata = arrow::util::test_util::parquet_test_data();
+ let path = format!("{testdata}/alltypes_tiny_pages.parquet");
+ let mut file = File::open(&path).await.unwrap();
+ let file_size = file.metadata().await.unwrap().len();
+ let metadata = ParquetMetaDataReader::new()
+ .with_page_indexes(true)
+ .load_and_finish(&mut file, file_size as usize)
+ .await
+ .unwrap();
+
+ let options = ArrowReaderOptions::new().with_page_index(true);
+ let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap();
+ let reader =
+ ParquetRecordBatchStreamBuilder::new_with_metadata(file, arrow_reader_metadata)
+ .build()
+ .unwrap();
+
+ let result = reader.try_collect::<Vec<_>>().await.unwrap();
+ assert_eq!(result.len(), 8);
+ }
+
+ #[tokio::test]
+ async fn empty_offset_index_doesnt_panic_in_column_chunks() {
+ use tempfile::TempDir;
+ use tokio::fs::File;
+ fn write_metadata_to_local_file(
+ metadata: ParquetMetaData,
+ file: impl AsRef<std::path::Path>,
+ ) {
+ use crate::file::metadata::ParquetMetaDataWriter;
+ use std::fs::File;
+ let file = File::create(file).unwrap();
+ ParquetMetaDataWriter::new(file, &metadata)
+ .finish()
+ .unwrap()
+ }
+
+ fn read_metadata_from_local_file(file: impl AsRef<std::path::Path>) -> ParquetMetaData {
+ use std::fs::File;
+ let file = File::open(file).unwrap();
+ ParquetMetaDataReader::new()
+ .with_page_indexes(true)
+ .parse_and_finish(&file)
+ .unwrap()
+ }
+
+ let testdata = arrow::util::test_util::parquet_test_data();
+ let path = format!("{testdata}/alltypes_plain.parquet");
+ let mut file = File::open(&path).await.unwrap();
+ let file_size = file.metadata().await.unwrap().len();
+ let metadata = ParquetMetaDataReader::new()
+ .with_page_indexes(true)
+ .load_and_finish(&mut file, file_size as usize)
+ .await
+ .unwrap();
+
+ let tempdir = TempDir::new().unwrap();
+ let metadata_path = tempdir.path().join("thrift_metadata.dat");
+ write_metadata_to_local_file(metadata, &metadata_path);
+ let metadata = read_metadata_from_local_file(&metadata_path);
+
+ let options = ArrowReaderOptions::new().with_page_index(true);
+ let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap();
+ let reader =
+ ParquetRecordBatchStreamBuilder::new_with_metadata(file, arrow_reader_metadata)
+ .build()
+ .unwrap();
+
+ // Panics here
+ let result = reader.try_collect::<Vec<_>>().await.unwrap();
+ assert_eq!(result.len(), 1);
+ }
}