You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by xu...@apache.org on 2025/04/09 03:02:01 UTC
(iceberg-rust) 01/01: refactor: Bump OpenDAL to 0.53
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch bump-opendal
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
commit fc30c32b8a68fe31f1fe93f19d32806e1a7ac68e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Apr 9 11:01:34 2025 +0800
refactor: Bump OpenDAL to 0.53
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
Cargo.lock | 11 ++++++-----
Cargo.toml | 4 ++--
crates/iceberg/src/io/file_io.rs | 3 ++-
crates/iceberg/src/io/storage.rs | 13 ++-----------
crates/iceberg/src/io/storage_s3.rs | 11 ++---------
5 files changed, 14 insertions(+), 28 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 1cc2cc42..619ae3ac 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1393,7 +1393,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
dependencies = [
"lazy_static",
- "windows-sys 0.48.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -4050,9 +4050,9 @@ checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc"
[[package]]
name = "opendal"
-version = "0.51.2"
+version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b1063ea459fa9e94584115743b06330f437902dd1d9f692b863ef1875a20548"
+checksum = "b5ebd1183902124c6b3ee0a9383683513dd8cca3d25a5d065593f969a44f979e"
dependencies = [
"anyhow",
"async-trait",
@@ -4066,7 +4066,6 @@ dependencies = [
"http 1.3.1",
"log",
"md-5",
- "once_cell",
"percent-encoding",
"quick-xml 0.36.2",
"reqsign",
@@ -6501,7 +6500,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9"
dependencies = [
"getrandom 0.3.1",
+ "js-sys",
"serde",
+ "wasm-bindgen",
]
[[package]]
@@ -6775,7 +6776,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.59.0",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index ed6380cb..aa8a2b61 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -74,7 +74,7 @@ mockito = "1"
murmur3 = "0.5.2"
num-bigint = "0.4.6"
once_cell = "1.20"
-opendal = "0.51.2"
+opendal = "0.53.0"
ordered-float = "4"
parquet = "54.2.0"
pilota = "0.11.2"
@@ -92,7 +92,7 @@ serde_json = "1.0.138"
serde_repr = "0.1.16"
serde_with = "3.4"
tempfile = "3.18"
-thrift = "0.17.0"
+thrift = "0.17.0"
tokio = { version = "1.44", default-features = false }
typed-builder = "0.20"
url = "2.5.4"
diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs
index 330f210c..9afaffd8 100644
--- a/crates/iceberg/src/io/file_io.rs
+++ b/crates/iceberg/src/io/file_io.rs
@@ -309,7 +309,8 @@ impl FileWrite for opendal::Writer {
}
async fn close(&mut self) -> crate::Result<()> {
- Ok(opendal::Writer::close(self).await?)
+ let _ = opendal::Writer::close(self).await?;
+ Ok(())
}
}
diff --git a/crates/iceberg/src/io/storage.rs b/crates/iceberg/src/io/storage.rs
index 605b3172..4ce0d37a 100644
--- a/crates/iceberg/src/io/storage.rs
+++ b/crates/iceberg/src/io/storage.rs
@@ -39,10 +39,6 @@ pub(crate) enum Storage {
/// s3 storage could have `s3://` and `s3a://`.
/// Storing the scheme string here to return the correct path.
scheme_str: String,
- /// uses the same client for one FileIO Storage.
- ///
- /// TODO: allow users to configure this client.
- client: reqwest::Client,
config: Arc<S3Config>,
},
#[cfg(feature = "storage-gcs")]
@@ -63,7 +59,6 @@ impl Storage {
#[cfg(feature = "storage-s3")]
Scheme::S3 => Ok(Self::S3 {
scheme_str,
- client: reqwest::Client::new(),
config: super::s3_config_parse(props)?.into(),
}),
#[cfg(feature = "storage-gcs")]
@@ -115,12 +110,8 @@ impl Storage {
}
}
#[cfg(feature = "storage-s3")]
- Storage::S3 {
- scheme_str,
- client,
- config,
- } => {
- let op = super::s3_config_build(client, config, path)?;
+ Storage::S3 { scheme_str, config } => {
+ let op = super::s3_config_build(config, path)?;
let op_info = op.info();
// Check prefix of s3 path.
diff --git a/crates/iceberg/src/io/storage_s3.rs b/crates/iceberg/src/io/storage_s3.rs
index 30f61fe3..8396888c 100644
--- a/crates/iceberg/src/io/storage_s3.rs
+++ b/crates/iceberg/src/io/storage_s3.rs
@@ -17,7 +17,6 @@
use std::collections::HashMap;
-use opendal::raw::HttpClient;
use opendal::services::S3Config;
use opendal::{Configurator, Operator};
use url::Url;
@@ -152,11 +151,7 @@ pub(crate) fn s3_config_parse(mut m: HashMap<String, String>) -> Result<S3Config
}
/// Build new opendal operator from give path.
-pub(crate) fn s3_config_build(
- client: &reqwest::Client,
- cfg: &S3Config,
- path: &str,
-) -> Result<Operator> {
+pub(crate) fn s3_config_build(cfg: &S3Config, path: &str) -> Result<Operator> {
let url = Url::parse(path)?;
let bucket = url.host_str().ok_or_else(|| {
Error::new(
@@ -169,9 +164,7 @@ pub(crate) fn s3_config_build(
.clone()
.into_builder()
// Set bucket name.
- .bucket(bucket)
- // Set http client we want to use.
- .http_client(HttpClient::with(client.clone()));
+ .bucket(bucket);
Ok(Operator::new(builder)?.finish())
}