You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/06/10 18:28:15 UTC
(datafusion) branch gh-readonly-queue/main/pr-22744-8bbc46004183c22bd6caf0b7492c21540248a084 created (now dae03ee062)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-22744-8bbc46004183c22bd6caf0b7492c21540248a084
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at dae03ee062 chore: Update to arrow/parquet 59.0.0 (#22744)
This branch includes the following new commits:
new dae03ee062 chore: Update to arrow/parquet 59.0.0 (#22744)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: chore: Update to arrow/parquet 59.0.0 (#22744)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-22744-8bbc46004183c22bd6caf0b7492c21540248a084
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit dae03ee062b2abf986de8df12ea82fb1578a2d99
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Wed Jun 10 14:27:53 2026 -0400
chore: Update to arrow/parquet 59.0.0 (#22744)
## Which issue does this PR close?
- related to https://github.com/apache/arrow-rs/issues/9110
## Rationale for this change
Update to latest version of arrow/parquet
## What changes are included in this PR?
1. Update to arrow/parquet 59.0.0
2. Adjust code for API differences
## Are these changes tested?
By CI
## Are there any user-facing changes?
New dependency
---
Cargo.lock | 99 ++++++++--------------
Cargo.toml | 18 ++--
datafusion-cli/src/main.rs | 12 +--
datafusion/common/src/file_options/mod.rs | 12 +--
.../common/src/file_options/parquet_writer.rs | 32 ++++---
.../core/tests/datasource/object_store_access.rs | 6 +-
.../core/tests/extension_types/pretty_printing.rs | 14 ++-
datafusion/core/tests/parquet/mod.rs | 10 +--
.../datasource-parquet/src/row_group_filter.rs | 25 ++----
datafusion/physical-expr-common/src/utils.rs | 8 +-
.../physical-expr/benches/in_list_strategy.rs | 2 +-
.../src/joins/sort_merge_join/tests.rs | 2 +-
datafusion/spark/src/function/hash/xxhash64.rs | 17 ++--
13 files changed, 114 insertions(+), 143 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index def7f88448..ed90dd25bd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -164,9 +164,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
+checksum = "ffaaa3e009861fd829d0a24dd6f115aa8e4634324bb092147d43baafe69ca4a7"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -187,9 +187,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
+checksum = "3ac95125e1d71c4a252b5a9c729aef111e80418f08aaa6dbabd1ba66918247fc"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -201,9 +201,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
+checksum = "0c60c79628e9a97cb90d7a0dc3e944f216a902f837d4ecabc14d524bddbbc137"
dependencies = [
"ahash",
"arrow-buffer",
@@ -220,9 +220,9 @@ dependencies = [
[[package]]
name = "arrow-avro"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376"
+checksum = "2835d67df2b69bf5de251ee6d289f85650d41b8169dcee0f950ea88747812c32"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -244,9 +244,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
+checksum = "6026f638c400e9878c1b1cc05c3cfd46fbf381285916ab408678701c1df46c1a"
dependencies = [
"bytes",
"half",
@@ -256,9 +256,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
+checksum = "c82c236c3caf8df5664284f3f1fbe89938852163998c3fdbf37e84ac220445e9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -278,9 +278,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
+checksum = "12714e5fb7954159af1e26d4e0d37108bcf1a2ad5ee5c5bf02a944d564d588b7"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -293,9 +293,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
+checksum = "7bd568aa70c4ec5947027b0d5caee94877433b661a0bb9e8ddceeeb5f0c9b1ab"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -306,9 +306,9 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704"
+checksum = "68365401e834743d708094927e2ca727a32d639fe900df04b936e07a36701b74"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -334,9 +334,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
+checksum = "e57ee4d470eab1a021bc4b63fa2b2c15d572892bf227b0a982d3b755a6c662b5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -350,9 +350,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
+checksum = "38f47e0e7a284e1f3707a780dc8cd5451b1614e9e398ea2d9ca03c7a2fe9a9ed"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -375,9 +375,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
+checksum = "a79cf73ad2eba8686ec2aa9bbf8671208e509025f166afc040cedbd94ffe4983"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -388,9 +388,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
+checksum = "cea0f7d8ed6182f14952761e2c0f989852d5aa334fcbc49f73a9f2247c25b879"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -401,9 +401,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
+checksum = "80b3e786a0dd9103acd583a6fb486dbf2f3268466cc0bd571dcf34cef231c1f1"
dependencies = [
"bitflags",
"serde",
@@ -413,9 +413,9 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
+checksum = "067a67e0361f6c31f4a7248759f36ca4ca71b187a941ed4d49da1c7d3d4db624"
dependencies = [
"ahash",
"arrow-array",
@@ -427,9 +427,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
+checksum = "99bc95847f3ff62a2b03d6f8ce2e3e78f01362060549a2a311898dd442f6256d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -3730,12 +3730,6 @@ dependencies = [
"serde_json",
]
-[[package]]
-name = "integer-encoding"
-version = "3.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
-
[[package]]
name = "ipnet"
version = "2.12.0"
@@ -4389,15 +4383,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
-[[package]]
-name = "ordered-float"
-version = "2.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
-dependencies = [
- "num-traits",
-]
-
[[package]]
name = "outref"
version = "0.5.2"
@@ -4445,9 +4430,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "58.3.0"
+version = "59.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
+checksum = "970dff83e97d953c827ae8176f6bf4e9f77bf62daacc01ec5df348ec5eacd913"
dependencies = [
"ahash",
"arrow-array",
@@ -4474,7 +4459,6 @@ dependencies = [
"seq-macro",
"simdutf8",
"snap",
- "thrift",
"tokio",
"twox-hash",
"zstd",
@@ -4834,7 +4818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck",
- "itertools 0.14.0",
+ "itertools 0.13.0",
"log",
"multimap",
"petgraph",
@@ -4853,7 +4837,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
- "itertools 0.14.0",
+ "itertools 0.13.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -6147,17 +6131,6 @@ dependencies = [
"cfg-if",
]
-[[package]]
-name = "thrift"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
-dependencies = [
- "byteorder",
- "integer-encoding",
- "ordered-float",
-]
-
[[package]]
name = "time"
version = "0.3.47"
diff --git a/Cargo.toml b/Cargo.toml
index 61b999d818..773a38ac50 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -89,30 +89,30 @@ version = "54.0.0"
#
# See for more details: https://github.com/rust-lang/cargo/issues/11329
apache-avro = { version = "0.21", default-features = false }
-arrow = { version = "58.3.0", features = [
+arrow = { version = "59.0.0", features = [
"prettyprint",
"chrono-tz",
] }
-arrow-avro = { version = "58.3.0", default-features = false, features = [
+arrow-avro = { version = "59.0.0", default-features = false, features = [
"deflate",
"snappy",
"zstd",
"bzip2",
"xz",
] }
-arrow-buffer = { version = "58.3.0", default-features = false }
-arrow-data = { version = "58.3.0", default-features = false }
-arrow-flight = { version = "58.3.0", features = [
+arrow-buffer = { version = "59.0.0", default-features = false }
+arrow-data = { version = "59.0.0", default-features = false }
+arrow-flight = { version = "59.0.0", features = [
"flight-sql-experimental",
] }
# Both codecs are required here to make sure that code paths like
# file-spilling have access to all compression codecs.
-arrow-ipc = { version = "58.3.0", default-features = false, features = [
+arrow-ipc = { version = "59.0.0", default-features = false, features = [
"lz4",
"zstd",
] }
-arrow-ord = { version = "58.3.0", default-features = false }
-arrow-schema = { version = "58.3.0", default-features = false }
+arrow-ord = { version = "59.0.0", default-features = false }
+arrow-schema = { version = "59.0.0", default-features = false }
async-trait = "0.1.89"
bigdecimal = "0.4.8"
bytes = "1.11"
@@ -178,7 +178,7 @@ memchr = "2.8.1"
num-traits = { version = "0.2" }
object_store = { version = "0.13.2", default-features = false }
parking_lot = "0.12"
-parquet = { version = "58.3.0", default-features = false, features = [
+parquet = { version = "59.0.0", default-features = false, features = [
"arrow",
"async",
"object_store",
diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs
index 935bf0a974..0d8ada1367 100644
--- a/datafusion-cli/src/main.rs
+++ b/datafusion-cli/src/main.rs
@@ -613,9 +613,9 @@ mod tests {
+-----------------------------------+-----------------+---------------------+------+------------------+
| filename | file_size_bytes | metadata_size_bytes | hits | extra |
+-----------------------------------+-----------------+---------------------+------+------------------+
- | alltypes_plain.parquet | 1851 | 8882 | 2 | page_index=false |
- | alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true |
- | lz4_raw_compressed_larger.parquet | 380836 | 1339 | 2 | page_index=false |
+ | alltypes_plain.parquet | 1851 | 8794 | 2 | page_index=false |
+ | alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |
+ | lz4_raw_compressed_larger.parquet | 380836 | 1331 | 2 | page_index=false |
+-----------------------------------+-----------------+---------------------+------+------------------+
");
@@ -644,9 +644,9 @@ mod tests {
+-----------------------------------+-----------------+---------------------+------+------------------+
| filename | file_size_bytes | metadata_size_bytes | hits | extra |
+-----------------------------------+-----------------+---------------------+------+------------------+
- | alltypes_plain.parquet | 1851 | 8882 | 5 | page_index=false |
- | alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true |
- | lz4_raw_compressed_larger.parquet | 380836 | 1339 | 3 | page_index=false |
+ | alltypes_plain.parquet | 1851 | 8794 | 5 | page_index=false |
+ | alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |
+ | lz4_raw_compressed_larger.parquet | 380836 | 1331 | 3 | page_index=false |
+-----------------------------------+-----------------+---------------------+------+------------------+
");
diff --git a/datafusion/common/src/file_options/mod.rs b/datafusion/common/src/file_options/mod.rs
index 5d2abd2317..97b4a44f03 100644
--- a/datafusion/common/src/file_options/mod.rs
+++ b/datafusion/common/src/file_options/mod.rs
@@ -114,14 +114,14 @@ mod tests {
properties
.bloom_filter_properties(&ColumnPath::from(""))
.expect("expected bloom properties!")
- .fpp,
+ .fpp(),
0.123
);
assert_eq!(
properties
.bloom_filter_properties(&ColumnPath::from(""))
.expect("expected bloom properties!")
- .ndv,
+ .ndv(),
123
);
@@ -242,7 +242,7 @@ mod tests {
properties
.bloom_filter_properties(&col1)
.expect("expected bloom properties!")
- .fpp,
+ .fpp(),
0.123
);
@@ -250,7 +250,7 @@ mod tests {
properties
.bloom_filter_properties(&col2_nested)
.expect("expected bloom properties!")
- .fpp,
+ .fpp(),
0.456
);
@@ -258,7 +258,7 @@ mod tests {
properties
.bloom_filter_properties(&col1)
.expect("expected bloom properties!")
- .ndv,
+ .ndv(),
123
);
@@ -266,7 +266,7 @@ mod tests {
properties
.bloom_filter_properties(&col2_nested)
.expect("expected bloom properties!")
- .ndv,
+ .ndv(),
456
);
diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rs
index a5b270a8f5..320bfcf33e 100644
--- a/datafusion/common/src/file_options/parquet_writer.rs
+++ b/datafusion/common/src/file_options/parquet_writer.rs
@@ -157,8 +157,8 @@ impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder {
}
if let Some(bloom_filter_ndv) = options.bloom_filter_ndv {
- builder =
- builder.set_column_bloom_filter_ndv(path.clone(), bloom_filter_ndv);
+ builder = builder
+ .set_column_bloom_filter_max_ndv(path.clone(), bloom_filter_ndv);
}
}
@@ -273,7 +273,7 @@ impl ParquetOptions {
builder = builder.set_bloom_filter_fpp(*bloom_filter_fpp);
};
if let Some(bloom_filter_ndv) = bloom_filter_ndv {
- builder = builder.set_bloom_filter_ndv(*bloom_filter_ndv);
+ builder = builder.set_bloom_filter_max_ndv(*bloom_filter_ndv);
};
if let Some(dictionary_enabled) = dictionary_enabled {
builder = builder.set_dictionary_enabled(*dictionary_enabled);
@@ -534,8 +534,8 @@ mod tests {
}
.into(),
),
- bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp),
- bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv),
+ bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp()),
+ bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv()),
}
}
@@ -830,10 +830,12 @@ mod tests {
);
assert_eq!(
default_writer_props.bloom_filter_properties(&"default".into()),
- Some(&BloomFilterProperties {
- fpp: 0.42,
- ndv: DEFAULT_BLOOM_FILTER_NDV
- }),
+ Some(
+ &BloomFilterProperties::builder()
+ .with_fpp(0.42)
+ .with_max_ndv(DEFAULT_BLOOM_FILTER_NDV)
+ .build()
+ ),
"should have only the fpp set, and the ndv at default",
);
}
@@ -937,7 +939,7 @@ mod tests {
// the WriterProperties::default, with only ndv set
let default_writer_props = WriterProperties::builder()
.set_bloom_filter_enabled(true)
- .set_bloom_filter_ndv(42)
+ .set_bloom_filter_max_ndv(42)
.build();
assert_eq!(
@@ -947,10 +949,12 @@ mod tests {
);
assert_eq!(
default_writer_props.bloom_filter_properties(&"default".into()),
- Some(&BloomFilterProperties {
- fpp: DEFAULT_BLOOM_FILTER_FPP,
- ndv: 42
- }),
+ Some(
+ &BloomFilterProperties::builder()
+ .with_fpp(DEFAULT_BLOOM_FILTER_FPP)
+ .with_max_ndv(42)
+ .build()
+ ),
"should have only the ndv set, and the fpp at default",
);
}
diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rs
index 83b84f6f92..25150ae284 100644
--- a/datafusion/core/tests/datasource/object_store_access.rs
+++ b/datafusion/core/tests/datasource/object_store_access.rs
@@ -904,7 +904,7 @@ async fn query_single_parquet_file_with_single_predicate() {
RequestCountingObjectStore()
Total Requests: 2
- GET (opts) path=parquet_table.parquet head=true
- - GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124
+ - GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124
"
);
}
@@ -928,8 +928,8 @@ async fn query_single_parquet_file_multi_row_groups_multiple_predicates() {
RequestCountingObjectStore()
Total Requests: 3
- GET (opts) path=parquet_table.parquet head=true
- - GET (ranges) path=parquet_table.parquet ranges=4-421,421-534,534-951,951-1064
- - GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124
+ - GET (ranges) path=parquet_table.parquet ranges=4-534,534-1064
+ - GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124
"
);
}
diff --git a/datafusion/core/tests/extension_types/pretty_printing.rs b/datafusion/core/tests/extension_types/pretty_printing.rs
index c0796887b8..f097b5bec9 100644
--- a/datafusion/core/tests/extension_types/pretty_printing.rs
+++ b/datafusion/core/tests/extension_types/pretty_printing.rs
@@ -40,10 +40,16 @@ async fn create_test_table() -> Result<DataFrame> {
// define data.
let batch = RecordBatch::try_new(
schema,
- vec![Arc::new(FixedSizeBinaryArray::from(vec![
- &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
- &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6],
- ]))],
+ vec![Arc::new(
+ FixedSizeBinaryArray::try_from_iter(
+ vec![
+ &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+ &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6],
+ ]
+ .into_iter(),
+ )
+ .unwrap(),
+ )],
)?;
let state = SessionStateBuilder::default()
diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs
index 0e936a79eb..12296f8498 100644
--- a/datafusion/core/tests/parquet/mod.rs
+++ b/datafusion/core/tests/parquet/mod.rs
@@ -726,11 +726,11 @@ fn make_bytearray_batch(
let name: StringArray = std::iter::repeat_n(Some(name), num_rows).collect();
let service_string: StringArray = string_values.iter().map(Some).collect();
let service_binary: BinaryArray = binary_values.iter().map(Some).collect();
- let service_fixedsize: FixedSizeBinaryArray = fixedsize_values
- .iter()
- .map(|value| Some(value.as_slice()))
- .collect::<Vec<_>>()
- .into();
+ let service_fixedsize = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+ fixedsize_values.iter().map(|value| Some(value.as_slice())),
+ 3,
+ )
+ .unwrap();
let service_large_binary: LargeBinaryArray =
large_binary_values.iter().map(Some).collect();
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs
index 07f4fe92cf..1e9b0636e5 100644
--- a/datafusion/datasource-parquet/src/row_group_filter.rs
+++ b/datafusion/datasource-parquet/src/row_group_filter.rs
@@ -955,10 +955,7 @@ mod tests {
let schema =
Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 2), false)]));
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32)
- .with_logical_type(LogicalType::Decimal {
- scale: 2,
- precision: 9,
- })
+ .with_logical_type(LogicalType::decimal(2, 9))
.with_scale(2)
.with_precision(9);
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1023,10 +1020,7 @@ mod tests {
Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 0), false)]));
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32)
- .with_logical_type(LogicalType::Decimal {
- scale: 0,
- precision: 9,
- })
+ .with_logical_type(LogicalType::decimal(0, 9))
.with_scale(0)
.with_precision(9);
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1118,10 +1112,7 @@ mod tests {
false,
)]));
let field = PrimitiveTypeField::new("c1", PhysicalType::INT64)
- .with_logical_type(LogicalType::Decimal {
- scale: 2,
- precision: 18,
- })
+ .with_logical_type(LogicalType::decimal(2, 18))
.with_scale(2)
.with_precision(18);
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1176,10 +1167,7 @@ mod tests {
false,
)]));
let field = PrimitiveTypeField::new("c1", PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_logical_type(LogicalType::Decimal {
- scale: 2,
- precision: 18,
- })
+ .with_logical_type(LogicalType::decimal(2, 18))
.with_scale(2)
.with_precision(18)
.with_byte_len(16);
@@ -1254,10 +1242,7 @@ mod tests {
false,
)]));
let field = PrimitiveTypeField::new("c1", PhysicalType::BYTE_ARRAY)
- .with_logical_type(LogicalType::Decimal {
- scale: 2,
- precision: 18,
- })
+ .with_logical_type(LogicalType::decimal(2, 18))
.with_scale(2)
.with_precision(18)
.with_byte_len(16);
diff --git a/datafusion/physical-expr-common/src/utils.rs b/datafusion/physical-expr-common/src/utils.rs
index e469885f83..117da23df2 100644
--- a/datafusion/physical-expr-common/src/utils.rs
+++ b/datafusion/physical-expr-common/src/utils.rs
@@ -614,11 +614,9 @@ mod tests {
#[test]
fn scatter_fixed_size_binary_test() -> Result<()> {
- let truthy = Arc::new(FixedSizeBinaryArray::from(vec![
- &[1u8, 2][..],
- &[3, 4][..],
- &[5, 6][..],
- ]));
+ let truthy = Arc::new(FixedSizeBinaryArray::try_from_iter(
+ vec![&[1u8, 2][..], &[3, 4][..], &[5, 6][..]].into_iter(),
+ )?);
let mask = BooleanArray::from(vec![true, false, true, false, true]);
let result = scatter(&mask, truthy.as_ref())?;
diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rs
index 5c4922fdcf..3eff1f5cf3 100644
--- a/datafusion/physical-expr/benches/in_list_strategy.rs
+++ b/datafusion/physical-expr/benches/in_list_strategy.rs
@@ -993,7 +993,7 @@ fn bench_fixed_size_binary_inner(
.collect();
let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect();
- let array = FixedSizeBinaryArray::from(refs);
+ let array = FixedSizeBinaryArray::try_from_iter(refs.into_iter()).unwrap();
let schema = Schema::new(vec![Field::new("a", array.data_type().clone(), true)]);
let exprs: Vec<_> = haystack
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index c4377b3189..b1fdf3ddab 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -175,7 +175,7 @@ fn build_fixed_size_binary_table(
let batch = RecordBatch::try_new(
Arc::new(schema),
vec![
- Arc::new(FixedSizeBinaryArray::from(a.1.clone())),
+ Arc::new(FixedSizeBinaryArray::try_from_iter(a.1.iter().copied()).unwrap()),
Arc::new(Int32Array::from(b.1.clone())),
Arc::new(Int32Array::from(c.1.clone())),
],
diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rs
index 5dca47bcb8..9d02a51b22 100644
--- a/datafusion/spark/src/function/hash/xxhash64.rs
+++ b/datafusion/spark/src/function/hash/xxhash64.rs
@@ -363,12 +363,17 @@ mod tests {
#[test]
fn test_xxhash64_fixed_size_binary() {
- let array = FixedSizeBinaryArray::from(vec![
- Some(&[0x01, 0x02, 0x03, 0x04][..]),
- Some(&[0x05, 0x06, 0x07, 0x08][..]),
- None,
- Some(&[0x00, 0x00, 0x00, 0x00][..]),
- ]);
+ let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+ vec![
+ Some(&[0x01, 0x02, 0x03, 0x04][..]),
+ Some(&[0x05, 0x06, 0x07, 0x08][..]),
+ None,
+ Some(&[0x00, 0x00, 0x00, 0x00][..]),
+ ]
+ .into_iter(),
+ 4,
+ )
+ .unwrap();
let array_ref: ArrayRef = Arc::new(array);
let mut hashes = vec![DEFAULT_SEED; 4];
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org