You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by et...@apache.org on 2025/09/08 23:18:29 UTC

(arrow-rs) branch gh5854_thrift_remodel updated (9fbe80a5b9 -> 9596775338)

This is an automated email from the ASF dual-hosted git repository.

etseidl pushed a change to branch gh5854_thrift_remodel
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git


    from 9fbe80a5b9 Merge remote-tracking branch 'origin/main' into gh5854_thrift_remodel
     new 64a59c3bff finish merge
     add bffad593c8 [Parquet] Write row group with async writer (#8262)
     add cd676cd85d [Variant] Add as_u* for Variant (#8284)
     add 2a8b18381e [Variant] Refactor `cast_to_variant` (#8235)
     add 8c80fe17ed Added arrow-avro enum mapping support for schema resolution (#8223)
     add 1c4d9252f9 feat: `SSLKEYLOGFILE` support for flight CLI (#8239)
     add 964805f5eb Update docstring comment for Writer::write() in writer.rs (#8267)
     add e5ead92bed Use apache/arrow-dotnet for integration test (#8295)
     add 0c7cb2ac3f Parquet: Do not compress v2 data page when compress is bad quality (#8257)
     add 911940f5a4 Added List and Struct Encoding to arrow-avro Writer (#8274)
     add 4b8cbe2c6d Add Decimal32 and Decimal64 support to arrow-avro Reader (#8255)
     add fb7d02ed09 [Variant] Support Shredded Objects in variant_get (take 2) (#8280)
     add d9a4b39815 Add `variant_experimental` feature to `parquet` crate (#8133)
     new 9596775338 Merge remote-tracking branch 'origin/main' into gh5854_thrift_remodel

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .github/workflows/integration.yml                  |    6 +
 .github/workflows/parquet.yml                      |    4 +-
 Cargo.toml                                         |    2 +-
 arrow-avro/Cargo.toml                              |    1 +
 arrow-avro/src/codec.rs                            |  418 +++--
 arrow-avro/src/reader/mod.rs                       |  235 ++-
 arrow-avro/src/reader/record.rs                    |  533 +++++--
 arrow-avro/src/schema.rs                           |  244 ++-
 arrow-avro/src/writer/encoder.rs                   |  777 ++++++++--
 arrow-avro/src/writer/format.rs                    |   22 +-
 arrow-avro/src/writer/mod.rs                       |  177 ++-
 arrow-avro/test/data/README.md                     |  147 ++
 arrow-avro/test/data/fixed256_decimal.avro         |  Bin 0 -> 1043 bytes
 .../test/data/fixed_length_decimal_legacy_32.avro  |  Bin 0 -> 378 bytes
 arrow-avro/test/data/int128_decimal.avro           |  Bin 0 -> 306 bytes
 arrow-avro/test/data/int256_decimal.avro           |  Bin 0 -> 380 bytes
 arrow-csv/src/writer.rs                            |    2 +-
 arrow-flight/src/bin/flight_sql_client.rs          |   14 +-
 parquet-variant-compute/src/cast_to_variant.rs     | 1639 ++++++++++----------
 parquet-variant-compute/src/variant_array.rs       |  341 +++-
 parquet-variant-compute/src/variant_get/mod.rs     | 1281 ++++++++++++++-
 .../src/variant_get/output/mod.rs                  |   83 +-
 .../src/variant_get/output/row_builder.rs          |  342 ++++
 parquet-variant/src/path.rs                        |   10 +-
 parquet-variant/src/variant.rs                     |  160 ++
 parquet/Cargo.toml                                 |    8 +-
 parquet/README.md                                  |    2 +
 parquet/src/arrow/async_writer/mod.rs              |   65 +-
 parquet/src/column/writer/mod.rs                   |   48 +-
 parquet/src/lib.rs                                 |   11 +
 parquet/src/variant.rs                             |  115 ++
 parquet/tests/arrow_reader/io/mod.rs               |    5 +-
 ...riant_integration.rs => variant_integration.rs} |    0
 33 files changed, 5137 insertions(+), 1555 deletions(-)
 create mode 100644 arrow-avro/test/data/README.md
 create mode 100644 arrow-avro/test/data/fixed256_decimal.avro
 create mode 100644 arrow-avro/test/data/fixed_length_decimal_legacy_32.avro
 create mode 100644 arrow-avro/test/data/int128_decimal.avro
 create mode 100644 arrow-avro/test/data/int256_decimal.avro
 create mode 100644 parquet-variant-compute/src/variant_get/output/row_builder.rs
 create mode 100644 parquet/src/variant.rs
 rename parquet/tests/{simple_variant_integration.rs => variant_integration.rs} (100%)


(arrow-rs) 01/02: finish merge

Posted by et...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

etseidl pushed a commit to branch gh5854_thrift_remodel
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git

commit 64a59c3bffe43f3f20c4e32334baf1920d3cb0e4
Author: Ed Seidl <et...@live.com>
AuthorDate: Mon Sep 8 16:14:29 2025 -0700

    finish merge
---
 parquet/tests/arrow_reader/io/mod.rs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/parquet/tests/arrow_reader/io/mod.rs b/parquet/tests/arrow_reader/io/mod.rs
index bfdb9467e2..749d075d71 100644
--- a/parquet/tests/arrow_reader/io/mod.rs
+++ b/parquet/tests/arrow_reader/io/mod.rs
@@ -287,10 +287,7 @@ impl TestRowGroups {
                     .map(|(col_idx, col_meta)| {
                         let column_name = col_meta.column_descr().name().to_string();
                         let page_locations = offset_index[rg_index][col_idx]
-                            .page_locations()
-                            .iter()
-                            .map(parquet::format::PageLocation::from)
-                            .collect();
+                            .page_locations().to_vec();
                         let dictionary_page_location = col_meta.dictionary_page_offset();
 
                         // We can find the byte range of the entire column chunk


(arrow-rs) 02/02: Merge remote-tracking branch 'origin/main' into gh5854_thrift_remodel

Posted by et...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

etseidl pushed a commit to branch gh5854_thrift_remodel
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git

commit 959677533878529a1790c3b9477848fa16a1fb73
Merge: 64a59c3bff d9a4b39815
Author: Ed Seidl <et...@live.com>
AuthorDate: Mon Sep 8 16:16:26 2025 -0700

    Merge remote-tracking branch 'origin/main' into gh5854_thrift_remodel

 .github/workflows/integration.yml                  |    6 +
 .github/workflows/parquet.yml                      |    4 +-
 Cargo.toml                                         |    2 +-
 arrow-avro/Cargo.toml                              |    1 +
 arrow-avro/src/codec.rs                            |  418 +++--
 arrow-avro/src/reader/mod.rs                       |  235 ++-
 arrow-avro/src/reader/record.rs                    |  533 +++++--
 arrow-avro/src/schema.rs                           |  244 ++-
 arrow-avro/src/writer/encoder.rs                   |  777 ++++++++--
 arrow-avro/src/writer/format.rs                    |   22 +-
 arrow-avro/src/writer/mod.rs                       |  177 ++-
 arrow-avro/test/data/README.md                     |  147 ++
 arrow-avro/test/data/fixed256_decimal.avro         |  Bin 0 -> 1043 bytes
 .../test/data/fixed_length_decimal_legacy_32.avro  |  Bin 0 -> 378 bytes
 arrow-avro/test/data/int128_decimal.avro           |  Bin 0 -> 306 bytes
 arrow-avro/test/data/int256_decimal.avro           |  Bin 0 -> 380 bytes
 arrow-csv/src/writer.rs                            |    2 +-
 arrow-flight/src/bin/flight_sql_client.rs          |   14 +-
 parquet-variant-compute/src/cast_to_variant.rs     | 1639 ++++++++++----------
 parquet-variant-compute/src/variant_array.rs       |  341 +++-
 parquet-variant-compute/src/variant_get/mod.rs     | 1281 ++++++++++++++-
 .../src/variant_get/output/mod.rs                  |   83 +-
 .../src/variant_get/output/row_builder.rs          |  342 ++++
 parquet-variant/src/path.rs                        |   10 +-
 parquet-variant/src/variant.rs                     |  160 ++
 parquet/Cargo.toml                                 |    8 +-
 parquet/README.md                                  |    2 +
 parquet/src/arrow/async_writer/mod.rs              |   65 +-
 parquet/src/column/writer/mod.rs                   |   48 +-
 parquet/src/lib.rs                                 |   11 +
 parquet/src/variant.rs                             |  115 ++
 ...riant_integration.rs => variant_integration.rs} |    0
 32 files changed, 5136 insertions(+), 1551 deletions(-)

diff --cc parquet/src/arrow/async_writer/mod.rs
index d9133801a1,4547f71274..09c0fc60d2
--- a/parquet/src/arrow/async_writer/mod.rs
+++ b/parquet/src/arrow/async_writer/mod.rs
@@@ -61,12 -61,11 +61,13 @@@ mod store
  pub use store::*;
  
  use crate::{
-     arrow::{arrow_writer::ArrowWriterOptions, ArrowWriter},
+     arrow::arrow_writer::{ArrowColumnChunk, ArrowColumnWriter, ArrowWriterOptions},
+     arrow::ArrowWriter,
      errors::{ParquetError, Result},
 -    file::{metadata::RowGroupMetaData, properties::WriterProperties},
 -    format::{FileMetaData, KeyValue},
 +    file::{
 +        metadata::{KeyValue, RowGroupMetaData},
 +        properties::WriterProperties,
 +    },
  };
  use arrow_array::RecordBatch;
  use arrow_schema::SchemaRef;
diff --cc parquet/src/lib.rs
index 74182d2650,1142a1c4a0..1ba731cb43
--- a/parquet/src/lib.rs
+++ b/parquet/src/lib.rs
@@@ -178,6 -186,7 +186,9 @@@ pub mod file
  pub mod record;
  pub mod schema;
  
 +mod parquet_macros;
 +mod parquet_thrift;
  pub mod thrift;
+ 
+ #[cfg(feature = "variant_experimental")]
+ pub mod variant;