You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/06/13 09:18:35 UTC
(opendal) 01/03: feat: Add HttpClientLayer
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch add-http-client-layer
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit ce9d6f3f40326364a2888fff9f8ccae84a8ebc2f
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Jun 13 17:10:56 2025 +0800
feat: Add HttpClientLayer
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/http_client.rs | 239 +++++++++++++++++++++++++++++++++++++++++
core/src/layers/mod.rs | 3 +
2 files changed, 242 insertions(+)
diff --git a/core/src/layers/http_client.rs b/core/src/layers/http_client.rs
new file mode 100644
index 000000000..111695937
--- /dev/null
+++ b/core/src/layers/http_client.rs
@@ -0,0 +1,239 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::*;
+
+/// Layer for replacing the default HTTP client with a custom one.
+///
+/// This layer allows users to provide their own HTTP client implementation
+/// by implementing the [`HttpFetch`] trait. This is useful when you need
+/// to customize HTTP behavior, add authentication, use a different HTTP
+/// client library, or apply custom middleware.
+///
+/// # Examples
+///
+/// ```no_run
+/// use opendal::layers::HttpClientLayer;
+/// use opendal::services;
+/// use opendal::Operator;
+/// use opendal::Result;
+/// use opendal::raw::HttpClient;
+///
+/// # fn main() -> Result<()> {
+/// // Create a custom HTTP client
+/// let custom_client = HttpClient::new()?;
+///
+/// let op = Operator::new(services::S3::default())?
+/// .layer(HttpClientLayer::new(custom_client))
+/// .finish();
+/// # Ok(())
+/// # }
+/// ```
+///
+/// # Custom HTTP Client Implementation
+///
+/// ```no_run
+/// use opendal::raw::{HttpFetch, HttpBody, Buffer};
+/// use http::{Request, Response};
+/// use opendal::Result;
+///
+/// struct CustomHttpClient {
+/// // Your custom HTTP client fields
+/// }
+///
+/// impl HttpFetch for CustomHttpClient {
+/// async fn fetch(&self, req: Request<Buffer>) -> Result<Response<HttpBody>> {
+/// // Your custom HTTP client implementation
+/// todo!()
+/// }
+/// }
+/// ```
+#[derive(Clone)]
+pub struct HttpClientLayer {
+ client: HttpClient,
+}
+
+impl HttpClientLayer {
+ /// Create a new `HttpClientLayer` with the given HTTP client.
+ ///
+ /// # Arguments
+ ///
+ /// * `client` - The HTTP client to use for all HTTP requests
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use opendal::layers::HttpClientLayer;
+ /// use opendal::raw::HttpClient;
+ /// use opendal::Result;
+ ///
+ /// # fn main() -> Result<()> {
+ /// let client = HttpClient::new()?;
+ /// let layer = HttpClientLayer::new(client);
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn new(client: HttpClient) -> Self {
+ Self { client }
+ }
+
+ /// Create a new `HttpClientLayer` with a custom HTTP fetcher.
+ ///
+ /// This is a convenience method for creating an HTTP client from
+ /// a custom implementation of [`HttpFetch`].
+ ///
+ /// # Arguments
+ ///
+ /// * `fetcher` - An implementation of the `HttpFetch` trait
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use opendal::layers::HttpClientLayer;
+ /// use opendal::raw::{HttpFetch, HttpBody, Buffer};
+ /// use opendal::Result;
+ ///
+ /// struct MyHttpClient;
+ ///
+ /// impl HttpFetch for MyHttpClient {
+ /// async fn fetch(&self, req: http::Request<Buffer>) -> Result<http::Response<HttpBody>> {
+ /// // Custom implementation
+ /// todo!()
+ /// }
+ /// }
+ ///
+ /// # fn main() {
+ /// let layer = HttpClientLayer::with_fetcher(MyHttpClient);
+ /// # }
+ /// ```
+ pub fn with_fetcher(fetcher: impl HttpFetch) -> Self {
+ Self {
+ client: HttpClient::with(fetcher),
+ }
+ }
+}
+
+impl<A: Access> Layer<A> for HttpClientLayer {
+ type LayeredAccess = HttpClientAccessor<A>;
+
+ fn layer(&self, inner: A) -> Self::LayeredAccess {
+ let info = inner.info();
+
+ // Replace the HTTP client with our custom one
+ info.update_http_client(|_| self.client.clone());
+
+ HttpClientAccessor { inner }
+ }
+}
+
+/// The accessor returned by [`HttpClientLayer`].
+///
+/// This accessor simply passes through all operations to the inner accessor,
+/// while the HTTP client replacement is handled at the layer level.
+#[derive(Debug, Clone)]
+pub struct HttpClientAccessor<A: Access> {
+ inner: A,
+}
+
+impl<A: Access> LayeredAccess for HttpClientAccessor<A> {
+ type Inner = A;
+ type Reader = A::Reader;
+ type Writer = A::Writer;
+ type Lister = A::Lister;
+ type Deleter = A::Deleter;
+
+ fn inner(&self) -> &Self::Inner {
+ &self.inner
+ }
+
+ async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
+ self.inner.create_dir(path, args).await
+ }
+
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+ self.inner.read(path, args).await
+ }
+
+ async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+ self.inner.write(path, args).await
+ }
+
+ async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
+ self.inner.copy(from, to, args).await
+ }
+
+ async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
+ self.inner.rename(from, to, args).await
+ }
+
+ async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+ self.inner.stat(path, args).await
+ }
+
+ async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+ self.inner.delete().await
+ }
+
+ async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+ self.inner.list(path, args).await
+ }
+
+ async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+ self.inner.presign(path, args).await
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::services;
+
+ #[tokio::test]
+ async fn test_http_client_layer() {
+ let layer = HttpClientLayer::new(HttpClient::new().unwrap());
+ let op = Operator::new(services::Memory::default())
+ .unwrap()
+ .layer(layer)
+ .finish();
+
+ // Basic test to ensure the layer doesn't break functionality
+ op.write("test", "data").await.unwrap();
+ let content = op.read("test").await.unwrap();
+ assert_eq!(content.to_bytes(), "data");
+ }
+
+ #[tokio::test]
+ async fn test_http_client_layer_with_fetcher() {
+ struct MockFetcher;
+
+ impl HttpFetch for MockFetcher {
+ async fn fetch(&self, _req: http::Request<Buffer>) -> Result<http::Response<HttpBody>> {
+ // For testing purposes, we just return an error since Memory service doesn't use HTTP
+ Err(Error::new(ErrorKind::Unexpected, "mock fetcher"))
+ }
+ }
+
+ let layer = HttpClientLayer::with_fetcher(MockFetcher);
+ let _op = Operator::new(services::Memory::default())
+ .unwrap()
+ .layer(layer)
+ .finish();
+
+ // The layer should be created successfully even with a custom fetcher
+ }
+}
diff --git a/core/src/layers/mod.rs b/core/src/layers/mod.rs
index dc8d79c29..50e58c280 100644
--- a/core/src/layers/mod.rs
+++ b/core/src/layers/mod.rs
@@ -125,3 +125,6 @@ mod correctness_check;
pub(crate) use correctness_check::CorrectnessCheckLayer;
mod capability_check;
pub use capability_check::CapabilityCheckLayer;
+
+mod http_client;
+pub use http_client::HttpClientLayer;