You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/07/18 07:57:11 UTC

[incubator-opendal] branch fix-security created (now 050a7d034)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a change to branch fix-security
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


      at 050a7d034 feat(core/http_util): Remove sensitive header like Set-Cookie

This branch includes the following new commits:

     new 050a7d034 feat(core/http_util): Remove sensitive header like Set-Cookie

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[incubator-opendal] 01/01: feat(core/http_util): Remove sensitive header like Set-Cookie

Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch fix-security
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit 050a7d034c428b2519ef1603215f16fd983622f9
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Jul 18 15:56:44 2023 +0800

    feat(core/http_util): Remove sensitive header like Set-Cookie
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/src/raw/http_util/error.rs             | 89 +++++++----------------------
 core/src/raw/http_util/mod.rs               |  3 +-
 core/src/services/azblob/error.rs           |  5 +-
 core/src/services/azdfs/error.rs            |  4 +-
 core/src/services/cos/error.rs              |  5 +-
 core/src/services/dropbox/error.rs          |  9 +--
 core/src/services/gcs/error.rs              |  4 +-
 core/src/services/gdrive/error.rs           |  7 ++-
 core/src/services/ghac/error.rs             |  7 ++-
 core/src/services/http/error.rs             |  7 ++-
 core/src/services/ipfs/error.rs             |  7 ++-
 core/src/services/ipmfs/error.rs            |  4 +-
 core/src/services/obs/error.rs              |  4 +-
 core/src/services/onedrive/error.rs         |  7 ++-
 core/src/services/oss/error.rs              |  4 +-
 core/src/services/s3/error.rs               |  9 +--
 core/src/services/supabase/error.rs         |  4 +-
 core/src/services/vercel_artifacts/error.rs |  7 ++-
 core/src/services/wasabi/error.rs           |  4 +-
 core/src/services/webdav/error.rs           |  7 ++-
 core/src/services/webhdfs/error.rs          |  4 +-
 21 files changed, 95 insertions(+), 106 deletions(-)

diff --git a/core/src/raw/http_util/error.rs b/core/src/raw/http_util/error.rs
index 3fed55c5e..34c416260 100644
--- a/core/src/raw/http_util/error.rs
+++ b/core/src/raw/http_util/error.rs
@@ -15,77 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::fmt::Display;
-use std::fmt::Formatter;
-
-use anyhow::anyhow;
 use http::response::Parts;
-use http::HeaderMap;
-use http::HeaderValue;
-use http::Response;
-use http::StatusCode;
+use http::Uri;
 
-use super::IncomingAsyncBody;
 use crate::Error;
 use crate::ErrorKind;
-use crate::Result;
-
-/// ErrorResponse carries HTTP status code, headers and body.
-///
-/// This struct should only be used to parse error response which is small.
-pub struct ErrorResponse {
-    parts: Parts,
-    body: Vec<u8>,
-}
-
-impl ErrorResponse {
-    /// Get http status code
-    pub fn status_code(&self) -> StatusCode {
-        self.parts.status
-    }
-
-    /// Get http headers
-    pub fn headers(&self) -> &HeaderMap<HeaderValue> {
-        &self.parts.headers
-    }
-
-    /// Get http error response body content (in bytes).
-    pub fn body(&self) -> &[u8] {
-        &self.body
-    }
-}
-
-impl Display for ErrorResponse {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "status code: {:?}, headers: {:?}, body: {:?}",
-            self.status_code(),
-            self.headers(),
-            String::from_utf8_lossy(self.body())
-        )
-    }
-}
-
-/// parse_error_response will parse response into `ErrorResponse`.
-///
-/// # NOTE
-///
-/// Please only use this for parsing error response hence it will read the
-/// entire body into memory.
-pub async fn parse_error_response(resp: Response<IncomingAsyncBody>) -> Result<ErrorResponse> {
-    let (parts, body) = resp.into_parts();
-    let bs = body.bytes().await.map_err(|err| {
-        Error::new(ErrorKind::Unexpected, "reading error response")
-            .with_operation("http_util::parse_error_response")
-            .set_source(anyhow!(err))
-    })?;
-
-    Ok(ErrorResponse {
-        parts,
-        body: bs.to_vec(),
-    })
-}
 
 /// Create a new error happened during building request.
 pub fn new_request_build_error(err: http::Error) -> Error {
@@ -111,3 +45,24 @@ pub fn new_request_sign_error(err: anyhow::Error) -> Error {
         .with_operation("reqsign::Sign")
         .set_source(err)
 }
+
+/// Add response context to error.
+///
+/// This helper function will:
+///
+/// - remove sensitive or useless headers from parts.
+/// - fetch uri if parts extensions contains `Uri`.
+pub fn with_error_response_context(mut err: Error, mut parts: Parts) -> Error {
+    if let Some(uri) = parts.extensions.get::<Uri>() {
+        err = err.with_context("uri", uri.to_string());
+    }
+
+    // The folloing headers may contains sensitive information.
+    parts.headers.remove("Set-Cookie");
+    parts.headers.remove("WWW-Authenticate");
+    parts.headers.remove("Proxy-Authenticate");
+
+    err = err.with_context("response", format!("{parts:?}"));
+
+    err
+}
diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs
index dc6822b88..2d6946db7 100644
--- a/core/src/raw/http_util/mod.rs
+++ b/core/src/raw/http_util/mod.rs
@@ -51,8 +51,7 @@ mod error;
 pub use error::new_request_build_error;
 pub use error::new_request_credential_error;
 pub use error::new_request_sign_error;
-pub use error::parse_error_response;
-pub use error::ErrorResponse;
+pub use error::with_error_response_context;
 
 mod bytes_range;
 pub use bytes_range::BytesRange;
diff --git a/core/src/services/azblob/error.rs b/core/src/services/azblob/error.rs
index 76eac0a30..80049d571 100644
--- a/core/src/services/azblob/error.rs
+++ b/core/src/services/azblob/error.rs
@@ -104,6 +104,7 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         Ok(azblob_err) => format!("{azblob_err:?}"),
         Err(_) => String::from_utf8_lossy(&bs).into_owned(),
     };
+
     // If there is no body here, fill with error code.
     if message.is_empty() {
         if let Some(v) = parts.headers.get("x-ms-error-code") {
@@ -119,7 +120,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         }
     }
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/azdfs/error.rs b/core/src/services/azdfs/error.rs
index 9acc81636..ad0cc94e1 100644
--- a/core/src/services/azdfs/error.rs
+++ b/core/src/services/azdfs/error.rs
@@ -95,7 +95,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         }
     }
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/cos/error.rs b/core/src/services/cos/error.rs
index ba451659e..e0d98920f 100644
--- a/core/src/services/cos/error.rs
+++ b/core/src/services/cos/error.rs
@@ -63,12 +63,13 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         Err(_) => String::from_utf8_lossy(&bs).into_owned(),
     };
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
     }
-
     Ok(err)
 }
 
diff --git a/core/src/services/dropbox/error.rs b/core/src/services/dropbox/error.rs
index b27722636..f25f298af 100644
--- a/core/src/services/dropbox/error.rs
+++ b/core/src/services/dropbox/error.rs
@@ -17,7 +17,6 @@
 
 use http::Response;
 use http::StatusCode;
-use http::Uri;
 use serde::Deserialize;
 
 use crate::raw::*;
@@ -55,16 +54,14 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
             parse_dropbox_error_summary(&dropbox_err.error_summary).unwrap_or((kind, retryable));
     }
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
     }
 
-    if let Some(uri) = parts.extensions.get::<Uri>() {
-        err = err.with_context("uri", uri.to_string());
-    }
-
     Ok(err)
 }
 
diff --git a/core/src/services/gcs/error.rs b/core/src/services/gcs/error.rs
index 3f2e0b688..2c5665ffd 100644
--- a/core/src/services/gcs/error.rs
+++ b/core/src/services/gcs/error.rs
@@ -72,7 +72,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         Err(_) => String::from_utf8_lossy(&bs).into_owned(),
     };
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/gdrive/error.rs b/core/src/services/gdrive/error.rs
index 6fe6b57d4..60de6ae0d 100644
--- a/core/src/services/gdrive/error.rs
+++ b/core/src/services/gdrive/error.rs
@@ -38,8 +38,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         _ => (ErrorKind::Unexpected, false),
     };
 
-    let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
-        .with_context("response", format!("{parts:?}"));
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/ghac/error.rs b/core/src/services/ghac/error.rs
index 005936b2d..598822f18 100644
--- a/core/src/services/ghac/error.rs
+++ b/core/src/services/ghac/error.rs
@@ -40,8 +40,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
     };
 
     let bs = body.bytes().await?;
-    let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
-        .with_context("response", format!("{parts:?}"));
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/http/error.rs b/core/src/services/http/error.rs
index aed2e13d6..c8afa09a9 100644
--- a/core/src/services/http/error.rs
+++ b/core/src/services/http/error.rs
@@ -41,8 +41,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         _ => (ErrorKind::Unexpected, false),
     };
 
-    let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
-        .with_context("response", format!("{parts:?}"));
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/ipfs/error.rs b/core/src/services/ipfs/error.rs
index 7020dc43d..1470b2098 100644
--- a/core/src/services/ipfs/error.rs
+++ b/core/src/services/ipfs/error.rs
@@ -40,8 +40,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         _ => (ErrorKind::Unexpected, false),
     };
 
-    let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
-        .with_context("response", format!("{parts:?}"));
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/ipmfs/error.rs b/core/src/services/ipmfs/error.rs
index 286eda36b..df6e7f475 100644
--- a/core/src/services/ipmfs/error.rs
+++ b/core/src/services/ipmfs/error.rs
@@ -73,7 +73,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         None => String::from_utf8_lossy(&bs).into_owned(),
     };
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/obs/error.rs b/core/src/services/obs/error.rs
index ddcca5f39..0c2081312 100644
--- a/core/src/services/obs/error.rs
+++ b/core/src/services/obs/error.rs
@@ -63,7 +63,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         Err(_) => String::from_utf8_lossy(&bs).into_owned(),
     };
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/onedrive/error.rs b/core/src/services/onedrive/error.rs
index 6fe6b57d4..60de6ae0d 100644
--- a/core/src/services/onedrive/error.rs
+++ b/core/src/services/onedrive/error.rs
@@ -38,8 +38,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         _ => (ErrorKind::Unexpected, false),
     };
 
-    let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
-        .with_context("response", format!("{parts:?}"));
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/oss/error.rs b/core/src/services/oss/error.rs
index 715ad744f..ecead41c4 100644
--- a/core/src/services/oss/error.rs
+++ b/core/src/services/oss/error.rs
@@ -59,7 +59,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         Err(_) => String::from_utf8_lossy(&bs).into_owned(),
     };
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/s3/error.rs b/core/src/services/s3/error.rs
index fce5c6d14..f68355487 100644
--- a/core/src/services/s3/error.rs
+++ b/core/src/services/s3/error.rs
@@ -17,7 +17,6 @@
 
 use bytes::Buf;
 use http::Response;
-use http::Uri;
 use quick_xml::de;
 use serde::Deserialize;
 
@@ -60,16 +59,14 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         (kind, retryable) = parse_s3_error_code(s3_err.code.as_str()).unwrap_or((kind, retryable));
     }
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
     }
 
-    if let Some(uri) = parts.extensions.get::<Uri>() {
-        err = err.with_context("uri", uri.to_string());
-    }
-
     Ok(err)
 }
 
diff --git a/core/src/services/supabase/error.rs b/core/src/services/supabase/error.rs
index 41c4225b2..58288959b 100644
--- a/core/src/services/supabase/error.rs
+++ b/core/src/services/supabase/error.rs
@@ -53,7 +53,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         })
         .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/vercel_artifacts/error.rs b/core/src/services/vercel_artifacts/error.rs
index 6fe6b57d4..60de6ae0d 100644
--- a/core/src/services/vercel_artifacts/error.rs
+++ b/core/src/services/vercel_artifacts/error.rs
@@ -38,8 +38,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         _ => (ErrorKind::Unexpected, false),
     };
 
-    let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
-        .with_context("response", format!("{parts:?}"));
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/wasabi/error.rs b/core/src/services/wasabi/error.rs
index 2e4423f48..74bb4aa2e 100644
--- a/core/src/services/wasabi/error.rs
+++ b/core/src/services/wasabi/error.rs
@@ -62,7 +62,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         (kind, retryable) = parse_wasabi_error_code(&wasabi_err.code).unwrap_or((kind, retryable));
     }
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/webdav/error.rs b/core/src/services/webdav/error.rs
index 472e7e30a..967dc19f5 100644
--- a/core/src/services/webdav/error.rs
+++ b/core/src/services/webdav/error.rs
@@ -41,8 +41,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
         _ => (ErrorKind::Unexpected, false),
     };
 
-    let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
-        .with_context("response", format!("{parts:?}"));
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();
diff --git a/core/src/services/webhdfs/error.rs b/core/src/services/webhdfs/error.rs
index eafd36bee..7964ea383 100644
--- a/core/src/services/webhdfs/error.rs
+++ b/core/src/services/webhdfs/error.rs
@@ -65,7 +65,9 @@ fn parse_error_msg(parts: Parts, body: &str) -> Result<Error> {
         Err(_) => body.to_owned(),
     };
 
-    let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+    let mut err = Error::new(kind, &message);
+
+    err = with_error_response_context(err, parts);
 
     if retryable {
         err = err.set_temporary();