You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by et...@apache.org on 2025/09/23 18:22:49 UTC
(arrow-rs) branch gh5854_thrift_remodel updated (3dbd42e516 -> 8dbfe4cd92)
This is an automated email from the ASF dual-hosted git repository.
etseidl pushed a change to branch gh5854_thrift_remodel
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
from 3dbd42e516 [thrift-remodel] Use new Thrift encoder/decoder for Parquet page headers (#8376)
add aed2f3b6a7 Add arrow-avro Reader support for Dense Union and Union resolution (Part 1) (#8348)
add f4840f6df1 [Variant] Implement new VariantValueArrayBuilder (#8360)
add 322745de67 Enable parallel writing across row groups when writing encrypted parquet (#8162)
add ae8e6c631a Update version to `56.2.0`, add changelog (#8372)
add e4d9942e4e [Geospatial]: Scaffolding for new `parquet-geospatial` crate (#8375)
add 138368cc9c fix: reset the offset of 'file_for_view' (#8381)
add 4431adf395 [Geospatial]: Add CI checks for `parquet-geospatial` crate (#8390)
add 7ac9db7f62 [Variant] [Shredding] feat: Support typed_access for Date32 (#8379)
add 18be750f18 Follow-up Improvements to Avro union handling (#8385)
add ca8e31e158 [Variant] Define new shred_variant function (#8366)
add 06c638fa2f Docs: Add more comments to the Parquet writer code (#8383)
add 83946594d6 [Variant] Fix NULL handling for shredded object fields (#8395)
add 7efb39556c Minor cleanup creating Schema (#8391)
add de84ff555c Propagate errors instead of panics: Replace usages of `new` with `try_new` for Array types (#8397)
add 13fb04118a perf: improve `GenericByteBuilder::append_array` to use SIMD for extending the offsets (#8388)
add 28ac4492dd Avro writer prefix support (#8371)
add 010d0e7db8 Add Arrow Variant Extension Type, remove `Array` impl for `VariantArray` and `ShreddedVariantFieldArray` (#8392)
add cdbbbf7509 Improve `Display` for `DataType` and `Field` (#8290)
add 78ab9d7800 [arrow-flight] Update prost-* crates to 0.14 (#8026)
add cdd15b81e7 Expose ReadPlan and ReadPlanBuilder (#8399)
add c9622533ff Fix red main by updating test (#8421)
add 63d0003e6e [Decimal] Add scale argument to validation functions to ensure accurate error logging (#8396)
add 07cb7f0588 Bump pyo3 to 0.26.0 (#8286)
add 75e5c5248f [Variant] mark metadata field as non-nullable (#8416)
new 8dbfe4cd92 Merge remote-tracking branch 'origin/main' into gh5854_thrift_remodel
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/integration.yml | 1 +
.../{parquet_derive.yml => parquet-geospatial.yml} | 31 +-
CHANGELOG-old.md | 132 +++
CHANGELOG.md | 240 +++---
Cargo.toml | 36 +-
arrow-array/src/array/fixed_size_list_array.rs | 2 +-
arrow-array/src/array/mod.rs | 6 +-
arrow-array/src/array/primitive_array.rs | 20 +-
arrow-array/src/builder/generic_bytes_builder.rs | 12 +-
arrow-array/src/builder/mod.rs | 8 +-
arrow-array/src/builder/struct_builder.rs | 6 +-
arrow-array/src/ffi.rs | 22 +-
arrow-array/src/record_batch.rs | 4 +-
arrow-array/src/types.rs | 47 +-
arrow-avro/benches/decoder.rs | 4 +-
arrow-avro/examples/read_with_utf8view.rs | 5 +-
arrow-avro/src/codec.rs | 495 +++++++++--
arrow-avro/src/reader/record.rs | 24 +-
arrow-avro/src/schema.rs | 412 ++++++++-
arrow-avro/src/writer/encoder.rs | 37 +-
arrow-avro/src/writer/format.rs | 42 +-
arrow-avro/src/writer/mod.rs | 107 ++-
arrow-cast/src/base64.rs | 6 +-
arrow-cast/src/cast/decimal.rs | 14 +-
arrow-cast/src/cast/dictionary.rs | 4 +-
arrow-cast/src/cast/list.rs | 14 +-
arrow-cast/src/cast/map.rs | 8 +-
arrow-cast/src/cast/mod.rs | 136 +--
arrow-cast/src/cast/string.rs | 2 +-
arrow-csv/src/reader/mod.rs | 2 +-
arrow-data/src/decimal.rs | 150 +++-
arrow-data/src/transform/run.rs | 4 +-
arrow-flight/Cargo.toml | 7 +-
arrow-flight/gen/Cargo.toml | 4 +-
arrow-flight/gen/src/main.rs | 8 +-
arrow-flight/src/arrow.flight.protocol.rs | 81 +-
arrow-flight/src/sql/arrow.flight.protocol.sql.rs | 64 +-
arrow-integration-test/src/lib.rs | 8 +-
arrow-integration-testing/Cargo.toml | 4 +-
.../flight_client_scenarios/auth_basic_proto.rs | 6 +-
.../flight_client_scenarios/integration_test.rs | 10 +-
arrow-json/src/lib.rs | 2 +-
arrow-ord/src/sort.rs | 4 +-
arrow-pyarrow-integration-testing/Cargo.toml | 2 +-
arrow-pyarrow-integration-testing/src/lib.rs | 14 +-
arrow-pyarrow-testing/Cargo.toml | 2 +-
arrow-pyarrow-testing/tests/pyarrow.rs | 16 +-
arrow-pyarrow/Cargo.toml | 2 +-
arrow-pyarrow/src/lib.rs | 35 +-
arrow-row/src/list.rs | 2 +-
arrow-row/src/run.rs | 4 +-
arrow-schema/src/datatype.rs | 24 +-
arrow-schema/src/datatype_display.rs | 247 ++++++
arrow-schema/src/datatype_parse.rs | 19 +-
arrow-schema/src/field.rs | 35 +-
arrow-schema/src/lib.rs | 1 +
arrow-schema/src/schema.rs | 17 +-
arrow-select/src/concat.rs | 2 +-
arrow-select/src/dictionary.rs | 2 +-
arrow-select/src/filter.rs | 7 +-
arrow-select/src/interleave.rs | 2 +-
arrow-select/src/take.rs | 6 +-
arrow-select/src/union_extract.rs | 2 +-
arrow-string/src/length.rs | 19 +-
arrow/src/util/data_gen.rs | 6 +-
arrow/tests/array_validation.rs | 11 +-
dev/release/update_change_log.sh | 4 +-
.../Cargo.toml | 18 +-
{parquet-variant => parquet-geospatial}/README.md | 19 +-
.../src/lib.rs | 18 +-
parquet-variant-compute/Cargo.toml | 1 +
parquet-variant-compute/benches/variant_kernels.rs | 2 +-
parquet-variant-compute/src/arrow_to_variant.rs | 4 +-
parquet-variant-compute/src/lib.rs | 6 +-
parquet-variant-compute/src/shred_variant.rs | 933 +++++++++++++++++++++
parquet-variant-compute/src/to_json.rs | 6 +-
parquet-variant-compute/src/variant_array.rs | 508 ++++++++---
.../src/variant_array_builder.rs | 243 +++++-
parquet-variant-compute/src/variant_get.rs | 344 ++++----
parquet-variant-compute/src/variant_to_arrow.rs | 192 +++--
parquet-variant/src/builder.rs | 4 +-
parquet-variant/src/variant.rs | 10 +-
parquet-variant/src/variant/object.rs | 12 +-
parquet/benches/arrow_reader_row_filter.rs | 2 +-
parquet/src/arrow/arrow_reader/mod.rs | 18 +-
parquet/src/arrow/arrow_reader/read_plan.rs | 20 +-
parquet/src/arrow/arrow_writer/mod.rs | 34 +-
parquet/src/arrow/async_writer/mod.rs | 67 +-
parquet/src/arrow/buffer/view_buffer.rs | 2 +-
parquet/src/basic.rs | 4 +-
parquet/src/column/writer/mod.rs | 12 +-
parquet/src/file/writer.rs | 45 +-
parquet/tests/encryption/encryption_async.rs | 367 +++++++-
parquet/tests/encryption/encryption_util.rs | 31 +-
parquet/tests/variant_integration.rs | 91 +-
95 files changed, 4486 insertions(+), 1239 deletions(-)
copy .github/workflows/{parquet_derive.yml => parquet-geospatial.yml} (69%)
create mode 100644 arrow-schema/src/datatype_display.rs
copy {parquet-variant-json => parquet-geospatial}/Cargo.toml (77%)
copy {parquet-variant => parquet-geospatial}/README.md (64%)
copy {parquet-variant-json => parquet-geospatial}/src/lib.rs (63%)
create mode 100644 parquet-variant-compute/src/shred_variant.rs
(arrow-rs) 01/01: Merge remote-tracking branch 'origin/main' into gh5854_thrift_remodel
Posted by et...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
etseidl pushed a commit to branch gh5854_thrift_remodel
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 8dbfe4cd9288f0b4d8c36a9cd8849a73ff2e9654
Merge: 3dbd42e516 75e5c5248f
Author: Ed Seidl <et...@live.com>
AuthorDate: Tue Sep 23 11:22:40 2025 -0700
Merge remote-tracking branch 'origin/main' into gh5854_thrift_remodel
.github/workflows/integration.yml | 1 +
.github/workflows/parquet-geospatial.yml | 79 ++
CHANGELOG-old.md | 132 +++
CHANGELOG.md | 240 +++---
Cargo.toml | 36 +-
arrow-array/src/array/fixed_size_list_array.rs | 2 +-
arrow-array/src/array/mod.rs | 6 +-
arrow-array/src/array/primitive_array.rs | 20 +-
arrow-array/src/builder/generic_bytes_builder.rs | 12 +-
arrow-array/src/builder/mod.rs | 8 +-
arrow-array/src/builder/struct_builder.rs | 6 +-
arrow-array/src/ffi.rs | 22 +-
arrow-array/src/record_batch.rs | 4 +-
arrow-array/src/types.rs | 47 +-
arrow-avro/benches/decoder.rs | 4 +-
arrow-avro/examples/read_with_utf8view.rs | 5 +-
arrow-avro/src/codec.rs | 495 +++++++++--
arrow-avro/src/reader/record.rs | 24 +-
arrow-avro/src/schema.rs | 412 ++++++++-
arrow-avro/src/writer/encoder.rs | 37 +-
arrow-avro/src/writer/format.rs | 42 +-
arrow-avro/src/writer/mod.rs | 107 ++-
arrow-cast/src/base64.rs | 6 +-
arrow-cast/src/cast/decimal.rs | 14 +-
arrow-cast/src/cast/dictionary.rs | 4 +-
arrow-cast/src/cast/list.rs | 14 +-
arrow-cast/src/cast/map.rs | 8 +-
arrow-cast/src/cast/mod.rs | 136 +--
arrow-cast/src/cast/string.rs | 2 +-
arrow-csv/src/reader/mod.rs | 2 +-
arrow-data/src/decimal.rs | 150 +++-
arrow-data/src/transform/run.rs | 4 +-
arrow-flight/Cargo.toml | 7 +-
arrow-flight/gen/Cargo.toml | 4 +-
arrow-flight/gen/src/main.rs | 8 +-
arrow-flight/src/arrow.flight.protocol.rs | 81 +-
arrow-flight/src/sql/arrow.flight.protocol.sql.rs | 64 +-
arrow-integration-test/src/lib.rs | 8 +-
arrow-integration-testing/Cargo.toml | 4 +-
.../flight_client_scenarios/auth_basic_proto.rs | 6 +-
.../flight_client_scenarios/integration_test.rs | 10 +-
arrow-json/src/lib.rs | 2 +-
arrow-ord/src/sort.rs | 4 +-
arrow-pyarrow-integration-testing/Cargo.toml | 2 +-
arrow-pyarrow-integration-testing/src/lib.rs | 14 +-
arrow-pyarrow-testing/Cargo.toml | 2 +-
arrow-pyarrow-testing/tests/pyarrow.rs | 16 +-
arrow-pyarrow/Cargo.toml | 2 +-
arrow-pyarrow/src/lib.rs | 35 +-
arrow-row/src/list.rs | 2 +-
arrow-row/src/run.rs | 4 +-
arrow-schema/src/datatype.rs | 24 +-
arrow-schema/src/datatype_display.rs | 247 ++++++
arrow-schema/src/datatype_parse.rs | 19 +-
arrow-schema/src/field.rs | 35 +-
arrow-schema/src/lib.rs | 1 +
arrow-schema/src/schema.rs | 17 +-
arrow-select/src/concat.rs | 2 +-
arrow-select/src/dictionary.rs | 2 +-
arrow-select/src/filter.rs | 7 +-
arrow-select/src/interleave.rs | 2 +-
arrow-select/src/take.rs | 6 +-
arrow-select/src/union_extract.rs | 2 +-
arrow-string/src/length.rs | 19 +-
arrow/src/util/data_gen.rs | 6 +-
arrow/tests/array_validation.rs | 11 +-
dev/release/update_change_log.sh | 4 +-
{arrow-pyarrow => parquet-geospatial}/Cargo.toml | 27 +-
parquet-geospatial/README.md | 43 +
parquet-geospatial/src/lib.rs | 28 +
parquet-variant-compute/Cargo.toml | 1 +
parquet-variant-compute/benches/variant_kernels.rs | 2 +-
parquet-variant-compute/src/arrow_to_variant.rs | 4 +-
parquet-variant-compute/src/lib.rs | 6 +-
parquet-variant-compute/src/shred_variant.rs | 933 +++++++++++++++++++++
parquet-variant-compute/src/to_json.rs | 6 +-
parquet-variant-compute/src/variant_array.rs | 508 ++++++++---
.../src/variant_array_builder.rs | 243 +++++-
parquet-variant-compute/src/variant_get.rs | 344 ++++----
parquet-variant-compute/src/variant_to_arrow.rs | 192 +++--
parquet-variant/src/builder.rs | 4 +-
parquet-variant/src/variant.rs | 10 +-
parquet-variant/src/variant/object.rs | 12 +-
parquet/benches/arrow_reader_row_filter.rs | 2 +-
parquet/src/arrow/arrow_reader/mod.rs | 18 +-
parquet/src/arrow/arrow_reader/read_plan.rs | 20 +-
parquet/src/arrow/arrow_writer/mod.rs | 34 +-
parquet/src/arrow/async_writer/mod.rs | 67 +-
parquet/src/arrow/buffer/view_buffer.rs | 2 +-
parquet/src/basic.rs | 4 +-
parquet/src/column/writer/mod.rs | 12 +-
parquet/src/file/writer.rs | 45 +-
parquet/tests/encryption/encryption_async.rs | 367 +++++++-
parquet/tests/encryption/encryption_util.rs | 31 +-
parquet/tests/variant_integration.rs | 91 +-
95 files changed, 4608 insertions(+), 1208 deletions(-)
diff --cc parquet/src/arrow/async_writer/mod.rs
index 09c0fc60d2,66ba6b87fe..e61b8f47c3
--- a/parquet/src/arrow/async_writer/mod.rs
+++ b/parquet/src/arrow/async_writer/mod.rs
@@@ -61,13 -61,11 +61,13 @@@ mod store
pub use store::*;
use crate::{
- arrow::arrow_writer::{ArrowColumnChunk, ArrowColumnWriter, ArrowWriterOptions},
+ arrow::arrow_writer::ArrowWriterOptions,
arrow::ArrowWriter,
errors::{ParquetError, Result},
- file::{metadata::RowGroupMetaData, properties::WriterProperties},
- format::{FileMetaData, KeyValue},
+ file::{
+ metadata::{KeyValue, RowGroupMetaData},
+ properties::WriterProperties,
+ },
};
use arrow_array::RecordBatch;
use arrow_schema::SchemaRef;
diff --cc parquet/src/file/writer.rs
index 71881b00ff,fa72b060ea..c8fba9bc5c
--- a/parquet/src/file/writer.rs
+++ b/parquet/src/file/writer.rs
@@@ -15,14 -15,12 +15,13 @@@
// specific language governing permissions and limitations
// under the License.
- //! Contains file writer API, and provides methods to write row groups and columns by
- //! using row group writers and column writers respectively.
+ //! [`SerializedFileWriter`]: Low level Parquet writer API
use crate::bloom_filter::Sbbf;
-use crate::format as parquet;
-use crate::format::{ColumnIndex, OffsetIndex};
-use crate::thrift::TSerializable;
+use crate::file::metadata::thrift_gen::PageHeader;
+use crate::file::page_index::index::Index;
+use crate::file::page_index::offset_index::OffsetIndexMetaData;
+use crate::parquet_thrift::{ThriftCompactOutputProtocol, WriteThrift};
use std::fmt::Debug;
use std::io::{BufWriter, IoSlice, Read};
use std::{io::Write, sync::Arc};