You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by xu...@apache.org on 2025/04/09 03:02:01 UTC

(iceberg-rust) 01/01: refactor: Bump OpenDAL to 0.53

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch bump-opendal
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git

commit fc30c32b8a68fe31f1fe93f19d32806e1a7ac68e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Apr 9 11:01:34 2025 +0800

    refactor: Bump OpenDAL to 0.53
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 Cargo.lock                          | 11 ++++++-----
 Cargo.toml                          |  4 ++--
 crates/iceberg/src/io/file_io.rs    |  3 ++-
 crates/iceberg/src/io/storage.rs    | 13 ++-----------
 crates/iceberg/src/io/storage_s3.rs | 11 ++---------
 5 files changed, 14 insertions(+), 28 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 1cc2cc42..619ae3ac 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1393,7 +1393,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
 dependencies = [
  "lazy_static",
- "windows-sys 0.48.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4050,9 +4050,9 @@ checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc"
 
 [[package]]
 name = "opendal"
-version = "0.51.2"
+version = "0.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b1063ea459fa9e94584115743b06330f437902dd1d9f692b863ef1875a20548"
+checksum = "b5ebd1183902124c6b3ee0a9383683513dd8cca3d25a5d065593f969a44f979e"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -4066,7 +4066,6 @@ dependencies = [
  "http 1.3.1",
  "log",
  "md-5",
- "once_cell",
  "percent-encoding",
  "quick-xml 0.36.2",
  "reqsign",
@@ -6501,7 +6500,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9"
 dependencies = [
  "getrandom 0.3.1",
+ "js-sys",
  "serde",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -6775,7 +6776,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index ed6380cb..aa8a2b61 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -74,7 +74,7 @@ mockito = "1"
 murmur3 = "0.5.2"
 num-bigint = "0.4.6"
 once_cell = "1.20"
-opendal = "0.51.2"
+opendal = "0.53.0"
 ordered-float = "4"
 parquet = "54.2.0"
 pilota = "0.11.2"
@@ -92,7 +92,7 @@ serde_json = "1.0.138"
 serde_repr = "0.1.16"
 serde_with = "3.4"
 tempfile = "3.18"
-thrift = "0.17.0"  
+thrift = "0.17.0"
 tokio = { version = "1.44", default-features = false }
 typed-builder = "0.20"
 url = "2.5.4"
diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs
index 330f210c..9afaffd8 100644
--- a/crates/iceberg/src/io/file_io.rs
+++ b/crates/iceberg/src/io/file_io.rs
@@ -309,7 +309,8 @@ impl FileWrite for opendal::Writer {
     }
 
     async fn close(&mut self) -> crate::Result<()> {
-        Ok(opendal::Writer::close(self).await?)
+        let _ = opendal::Writer::close(self).await?;
+        Ok(())
     }
 }
 
diff --git a/crates/iceberg/src/io/storage.rs b/crates/iceberg/src/io/storage.rs
index 605b3172..4ce0d37a 100644
--- a/crates/iceberg/src/io/storage.rs
+++ b/crates/iceberg/src/io/storage.rs
@@ -39,10 +39,6 @@ pub(crate) enum Storage {
         /// s3 storage could have `s3://` and `s3a://`.
         /// Storing the scheme string here to return the correct path.
         scheme_str: String,
-        /// uses the same client for one FileIO Storage.
-        ///
-        /// TODO: allow users to configure this client.
-        client: reqwest::Client,
         config: Arc<S3Config>,
     },
     #[cfg(feature = "storage-gcs")]
@@ -63,7 +59,6 @@ impl Storage {
             #[cfg(feature = "storage-s3")]
             Scheme::S3 => Ok(Self::S3 {
                 scheme_str,
-                client: reqwest::Client::new(),
                 config: super::s3_config_parse(props)?.into(),
             }),
             #[cfg(feature = "storage-gcs")]
@@ -115,12 +110,8 @@ impl Storage {
                 }
             }
             #[cfg(feature = "storage-s3")]
-            Storage::S3 {
-                scheme_str,
-                client,
-                config,
-            } => {
-                let op = super::s3_config_build(client, config, path)?;
+            Storage::S3 { scheme_str, config } => {
+                let op = super::s3_config_build(config, path)?;
                 let op_info = op.info();
 
                 // Check prefix of s3 path.
diff --git a/crates/iceberg/src/io/storage_s3.rs b/crates/iceberg/src/io/storage_s3.rs
index 30f61fe3..8396888c 100644
--- a/crates/iceberg/src/io/storage_s3.rs
+++ b/crates/iceberg/src/io/storage_s3.rs
@@ -17,7 +17,6 @@
 
 use std::collections::HashMap;
 
-use opendal::raw::HttpClient;
 use opendal::services::S3Config;
 use opendal::{Configurator, Operator};
 use url::Url;
@@ -152,11 +151,7 @@ pub(crate) fn s3_config_parse(mut m: HashMap<String, String>) -> Result<S3Config
 }
 
 /// Build new opendal operator from give path.
-pub(crate) fn s3_config_build(
-    client: &reqwest::Client,
-    cfg: &S3Config,
-    path: &str,
-) -> Result<Operator> {
+pub(crate) fn s3_config_build(cfg: &S3Config, path: &str) -> Result<Operator> {
     let url = Url::parse(path)?;
     let bucket = url.host_str().ok_or_else(|| {
         Error::new(
@@ -169,9 +164,7 @@ pub(crate) fn s3_config_build(
         .clone()
         .into_builder()
         // Set bucket name.
-        .bucket(bucket)
-        // Set http client we want to use.
-        .http_client(HttpClient::with(client.clone()));
+        .bucket(bucket);
 
     Ok(Operator::new(builder)?.finish())
 }