You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "alamb (via GitHub)" <gi...@apache.org> on 2023/08/18 11:12:55 UTC
[GitHub] [arrow-rs] alamb commented on a diff in pull request #4680: Parquet doc tweaks
alamb commented on code in PR #4680:
URL: https://github.com/apache/arrow-rs/pull/4680#discussion_r1298329066
##########
parquet/src/arrow/async_reader/mod.rs:
##########
@@ -263,6 +265,39 @@ impl<T: AsyncFileReader + Send + 'static> ParquetRecordBatchStreamBuilder<T> {
///
/// This allows loading metadata once and using it to create multiple builders with
/// potentially different settings
+ ///
+ /// ```
+ /// # use std::fs::metadata;
+ /// # use std::sync::Arc;
+ /// # use bytes::Bytes;
+ /// # use arrow_array::{Int32Array, RecordBatch};
+ /// # use arrow_schema::{DataType, Field, Schema};
+ /// # use parquet::arrow::arrow_reader::ArrowReaderMetadata;
+ /// # use parquet::arrow::{ArrowWriter, ParquetRecordBatchStreamBuilder};
+ /// # use tempfile::tempfile;
+ /// # use futures::StreamExt;
+ /// # #[tokio::main(flavor="current_thread")]
+ /// # async fn main() {
+ /// #
+ /// let mut file = tempfile().unwrap();
+ /// # let schema = Arc::new(Schema::new(vec![Field::new("i32", DataType::Int32, false)]));
+ /// # let mut writer = ArrowWriter::try_new(&mut file, schema.clone(), None).unwrap();
+ /// # let batch = RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
+ /// # writer.write(&batch).unwrap();
+ /// # writer.close().unwrap();
+ /// #
+ /// let mut file = tokio::fs::File::from_std(file);
+ /// let meta = ArrowReaderMetadata::load_async(&mut file, Default::default()).await.unwrap();
+ /// let mut a = ParquetRecordBatchStreamBuilder::new_with_metadata(
+ /// file.try_clone().await.unwrap(),
+ /// meta.clone()
+ /// ).build().unwrap();
+ /// let mut b = ParquetRecordBatchStreamBuilder::new_with_metadata(file, meta).build().unwrap();
Review Comment:
👍
##########
parquet/src/arrow/async_reader/mod.rs:
##########
@@ -263,6 +265,39 @@ impl<T: AsyncFileReader + Send + 'static> ParquetRecordBatchStreamBuilder<T> {
///
/// This allows loading metadata once and using it to create multiple builders with
/// potentially different settings
+ ///
+ /// ```
+ /// # use std::fs::metadata;
+ /// # use std::sync::Arc;
+ /// # use bytes::Bytes;
+ /// # use arrow_array::{Int32Array, RecordBatch};
+ /// # use arrow_schema::{DataType, Field, Schema};
+ /// # use parquet::arrow::arrow_reader::ArrowReaderMetadata;
+ /// # use parquet::arrow::{ArrowWriter, ParquetRecordBatchStreamBuilder};
+ /// # use tempfile::tempfile;
+ /// # use futures::StreamExt;
+ /// # #[tokio::main(flavor="current_thread")]
+ /// # async fn main() {
+ /// #
+ /// let mut file = tempfile().unwrap();
+ /// # let schema = Arc::new(Schema::new(vec![Field::new("i32", DataType::Int32, false)]));
+ /// # let mut writer = ArrowWriter::try_new(&mut file, schema.clone(), None).unwrap();
+ /// # let batch = RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
+ /// # writer.write(&batch).unwrap();
+ /// # writer.close().unwrap();
+ /// #
+ /// let mut file = tokio::fs::File::from_std(file);
+ /// let meta = ArrowReaderMetadata::load_async(&mut file, Default::default()).await.unwrap();
+ /// let mut a = ParquetRecordBatchStreamBuilder::new_with_metadata(
+ /// file.try_clone().await.unwrap(),
+ /// meta.clone()
+ /// ).build().unwrap();
+ /// let mut b = ParquetRecordBatchStreamBuilder::new_with_metadata(file, meta).build().unwrap();
Review Comment:
👍
##########
parquet/src/arrow/mod.rs:
##########
@@ -25,47 +25,39 @@
//!# Example of writing Arrow record batch to Parquet file
//!
//!```rust
-//! use arrow_array::{Int32Array, ArrayRef};
-//! use arrow_array::RecordBatch;
-//! use parquet::arrow::arrow_writer::ArrowWriter;
-//! use parquet::file::properties::WriterProperties;
-//! use std::fs::File;
-//! use std::sync::Arc;
+//! # use arrow_array::{Int32Array, ArrayRef};
+//! # use arrow_array::RecordBatch;
+//! # use parquet::arrow::arrow_writer::ArrowWriter;
+//! # use parquet::file::properties::WriterProperties;
+//! # use tempfile::tempfile;
+//! # use std::sync::Arc;
+//! # use parquet::basic::Compression;
//! let ids = Int32Array::from(vec![1, 2, 3, 4]);
//! let vals = Int32Array::from(vec![5, 6, 7, 8]);
//! let batch = RecordBatch::try_from_iter(vec![
//! ("id", Arc::new(ids) as ArrayRef),
//! ("val", Arc::new(vals) as ArrayRef),
//! ]).unwrap();
//!
-//! let file = File::create("data.parquet").unwrap();
+//! let file = tempfile().unwrap();
//!
-//! let mut writer = ArrowWriter::try_new(file, batch.schema(), None).unwrap();
+//! let props = WriterProperties::builder()
Review Comment:
Maybe it would help to draw the readers's attention to what this is doing
##########
parquet/src/arrow/mod.rs:
##########
@@ -25,47 +25,39 @@
//!# Example of writing Arrow record batch to Parquet file
//!
//!```rust
-//! use arrow_array::{Int32Array, ArrayRef};
-//! use arrow_array::RecordBatch;
-//! use parquet::arrow::arrow_writer::ArrowWriter;
-//! use parquet::file::properties::WriterProperties;
-//! use std::fs::File;
-//! use std::sync::Arc;
+//! # use arrow_array::{Int32Array, ArrayRef};
+//! # use arrow_array::RecordBatch;
+//! # use parquet::arrow::arrow_writer::ArrowWriter;
+//! # use parquet::file::properties::WriterProperties;
+//! # use tempfile::tempfile;
+//! # use std::sync::Arc;
+//! # use parquet::basic::Compression;
//! let ids = Int32Array::from(vec![1, 2, 3, 4]);
//! let vals = Int32Array::from(vec![5, 6, 7, 8]);
//! let batch = RecordBatch::try_from_iter(vec![
//! ("id", Arc::new(ids) as ArrayRef),
//! ("val", Arc::new(vals) as ArrayRef),
//! ]).unwrap();
//!
-//! let file = File::create("data.parquet").unwrap();
+//! let file = tempfile().unwrap();
//!
-//! let mut writer = ArrowWriter::try_new(file, batch.schema(), None).unwrap();
+//! let props = WriterProperties::builder()
Review Comment:
```suggestion
//! // WriterProperties can be used to set Parquet file options
//! let props = WriterProperties::builder()
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org