You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by et...@apache.org on 2025/10/14 14:24:51 UTC

(arrow-rs) branch main updated: [parquet] Add a sync fn to ArrowWriter that flushes Writer (#8586)

This is an automated email from the ASF dual-hosted git repository.

etseidl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git


The following commit(s) were added to refs/heads/main by this push:
     new 615a144687 [parquet] Add a sync fn to ArrowWriter that flushes Writer (#8586)
615a144687 is described below

commit 615a1446874bd817aa1af4e1508d43d8420356ae
Author: Piotr Srebrny <sr...@gmail.com>
AuthorDate: Tue Oct 14 16:24:45 2025 +0200

    [parquet] Add a sync fn to ArrowWriter that flushes Writer (#8586)
    
    # Which issue does this PR close?
    
    - Closes #8534.
    
    # Are these changes tested?
    
    Changes where not tested and should be covered by existing.
    
    # Are there any user-facing changes?
    
    * added the `ArrowWriter::sync()` function that flushes the underlying
    writer
    * added the `SerializedFileWriter::flush()` function that flushes the
    underlying writer
    
    ---------
    
    Co-authored-by: Ed Seidl <et...@users.noreply.github.com>
---
 parquet/src/arrow/arrow_writer/mod.rs | 8 ++++++++
 parquet/src/file/writer.rs            | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs
index 020f99c276..3625c3e844 100644
--- a/parquet/src/arrow/arrow_writer/mod.rs
+++ b/parquet/src/arrow/arrow_writer/mod.rs
@@ -354,7 +354,15 @@ impl<W: Write + Send> ArrowWriter<W> {
         self.writer.write_all(buf)
     }
 
+    /// Flushes underlying writer
+    pub fn sync(&mut self) -> std::io::Result<()> {
+        self.writer.flush()
+    }
+
     /// Flushes all buffered rows into a new row group
+    ///
+    /// Note the underlying writer is not flushed with this call.
+    /// If this is a desired behavior, please call [`ArrowWriter::sync`].
     pub fn flush(&mut self) -> Result<()> {
         let in_progress = match self.in_progress.take() {
             Some(in_progress) => in_progress,
diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs
index d261d9ec65..4972fbf79b 100644
--- a/parquet/src/file/writer.rs
+++ b/parquet/src/file/writer.rs
@@ -414,6 +414,11 @@ impl<W: Write + Send> SerializedFileWriter<W> {
         self.buf.write_all(buf)
     }
 
+    /// Flushes underlying writer
+    pub fn flush(&mut self) -> std::io::Result<()> {
+        self.buf.flush()
+    }
+
     /// Returns a mutable reference to the underlying writer.
     ///
     /// **Warning**: if you write directly to this writer, you will skip