You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/07/18 07:57:12 UTC
[incubator-opendal] 01/01: feat(core/http_util): Remove sensitive header like Set-Cookie
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fix-security
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 050a7d034c428b2519ef1603215f16fd983622f9
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Jul 18 15:56:44 2023 +0800
feat(core/http_util): Remove sensitive header like Set-Cookie
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/http_util/error.rs | 89 +++++++----------------------
core/src/raw/http_util/mod.rs | 3 +-
core/src/services/azblob/error.rs | 5 +-
core/src/services/azdfs/error.rs | 4 +-
core/src/services/cos/error.rs | 5 +-
core/src/services/dropbox/error.rs | 9 +--
core/src/services/gcs/error.rs | 4 +-
core/src/services/gdrive/error.rs | 7 ++-
core/src/services/ghac/error.rs | 7 ++-
core/src/services/http/error.rs | 7 ++-
core/src/services/ipfs/error.rs | 7 ++-
core/src/services/ipmfs/error.rs | 4 +-
core/src/services/obs/error.rs | 4 +-
core/src/services/onedrive/error.rs | 7 ++-
core/src/services/oss/error.rs | 4 +-
core/src/services/s3/error.rs | 9 +--
core/src/services/supabase/error.rs | 4 +-
core/src/services/vercel_artifacts/error.rs | 7 ++-
core/src/services/wasabi/error.rs | 4 +-
core/src/services/webdav/error.rs | 7 ++-
core/src/services/webhdfs/error.rs | 4 +-
21 files changed, 95 insertions(+), 106 deletions(-)
diff --git a/core/src/raw/http_util/error.rs b/core/src/raw/http_util/error.rs
index 3fed55c5e..34c416260 100644
--- a/core/src/raw/http_util/error.rs
+++ b/core/src/raw/http_util/error.rs
@@ -15,77 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-use std::fmt::Display;
-use std::fmt::Formatter;
-
-use anyhow::anyhow;
use http::response::Parts;
-use http::HeaderMap;
-use http::HeaderValue;
-use http::Response;
-use http::StatusCode;
+use http::Uri;
-use super::IncomingAsyncBody;
use crate::Error;
use crate::ErrorKind;
-use crate::Result;
-
-/// ErrorResponse carries HTTP status code, headers and body.
-///
-/// This struct should only be used to parse error response which is small.
-pub struct ErrorResponse {
- parts: Parts,
- body: Vec<u8>,
-}
-
-impl ErrorResponse {
- /// Get http status code
- pub fn status_code(&self) -> StatusCode {
- self.parts.status
- }
-
- /// Get http headers
- pub fn headers(&self) -> &HeaderMap<HeaderValue> {
- &self.parts.headers
- }
-
- /// Get http error response body content (in bytes).
- pub fn body(&self) -> &[u8] {
- &self.body
- }
-}
-
-impl Display for ErrorResponse {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- write!(
- f,
- "status code: {:?}, headers: {:?}, body: {:?}",
- self.status_code(),
- self.headers(),
- String::from_utf8_lossy(self.body())
- )
- }
-}
-
-/// parse_error_response will parse response into `ErrorResponse`.
-///
-/// # NOTE
-///
-/// Please only use this for parsing error response hence it will read the
-/// entire body into memory.
-pub async fn parse_error_response(resp: Response<IncomingAsyncBody>) -> Result<ErrorResponse> {
- let (parts, body) = resp.into_parts();
- let bs = body.bytes().await.map_err(|err| {
- Error::new(ErrorKind::Unexpected, "reading error response")
- .with_operation("http_util::parse_error_response")
- .set_source(anyhow!(err))
- })?;
-
- Ok(ErrorResponse {
- parts,
- body: bs.to_vec(),
- })
-}
/// Create a new error happened during building request.
pub fn new_request_build_error(err: http::Error) -> Error {
@@ -111,3 +45,24 @@ pub fn new_request_sign_error(err: anyhow::Error) -> Error {
.with_operation("reqsign::Sign")
.set_source(err)
}
+
+/// Add response context to error.
+///
+/// This helper function will:
+///
+/// - remove sensitive or useless headers from parts.
+/// - fetch uri if parts extensions contains `Uri`.
+pub fn with_error_response_context(mut err: Error, mut parts: Parts) -> Error {
+ if let Some(uri) = parts.extensions.get::<Uri>() {
+ err = err.with_context("uri", uri.to_string());
+ }
+
+ // The folloing headers may contains sensitive information.
+ parts.headers.remove("Set-Cookie");
+ parts.headers.remove("WWW-Authenticate");
+ parts.headers.remove("Proxy-Authenticate");
+
+ err = err.with_context("response", format!("{parts:?}"));
+
+ err
+}
diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs
index dc6822b88..2d6946db7 100644
--- a/core/src/raw/http_util/mod.rs
+++ b/core/src/raw/http_util/mod.rs
@@ -51,8 +51,7 @@ mod error;
pub use error::new_request_build_error;
pub use error::new_request_credential_error;
pub use error::new_request_sign_error;
-pub use error::parse_error_response;
-pub use error::ErrorResponse;
+pub use error::with_error_response_context;
mod bytes_range;
pub use bytes_range::BytesRange;
diff --git a/core/src/services/azblob/error.rs b/core/src/services/azblob/error.rs
index 76eac0a30..80049d571 100644
--- a/core/src/services/azblob/error.rs
+++ b/core/src/services/azblob/error.rs
@@ -104,6 +104,7 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
Ok(azblob_err) => format!("{azblob_err:?}"),
Err(_) => String::from_utf8_lossy(&bs).into_owned(),
};
+
// If there is no body here, fill with error code.
if message.is_empty() {
if let Some(v) = parts.headers.get("x-ms-error-code") {
@@ -119,7 +120,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
}
}
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/azdfs/error.rs b/core/src/services/azdfs/error.rs
index 9acc81636..ad0cc94e1 100644
--- a/core/src/services/azdfs/error.rs
+++ b/core/src/services/azdfs/error.rs
@@ -95,7 +95,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
}
}
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/cos/error.rs b/core/src/services/cos/error.rs
index ba451659e..e0d98920f 100644
--- a/core/src/services/cos/error.rs
+++ b/core/src/services/cos/error.rs
@@ -63,12 +63,13 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
Err(_) => String::from_utf8_lossy(&bs).into_owned(),
};
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
}
-
Ok(err)
}
diff --git a/core/src/services/dropbox/error.rs b/core/src/services/dropbox/error.rs
index b27722636..f25f298af 100644
--- a/core/src/services/dropbox/error.rs
+++ b/core/src/services/dropbox/error.rs
@@ -17,7 +17,6 @@
use http::Response;
use http::StatusCode;
-use http::Uri;
use serde::Deserialize;
use crate::raw::*;
@@ -55,16 +54,14 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
parse_dropbox_error_summary(&dropbox_err.error_summary).unwrap_or((kind, retryable));
}
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
}
- if let Some(uri) = parts.extensions.get::<Uri>() {
- err = err.with_context("uri", uri.to_string());
- }
-
Ok(err)
}
diff --git a/core/src/services/gcs/error.rs b/core/src/services/gcs/error.rs
index 3f2e0b688..2c5665ffd 100644
--- a/core/src/services/gcs/error.rs
+++ b/core/src/services/gcs/error.rs
@@ -72,7 +72,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
Err(_) => String::from_utf8_lossy(&bs).into_owned(),
};
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/gdrive/error.rs b/core/src/services/gdrive/error.rs
index 6fe6b57d4..60de6ae0d 100644
--- a/core/src/services/gdrive/error.rs
+++ b/core/src/services/gdrive/error.rs
@@ -38,8 +38,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
_ => (ErrorKind::Unexpected, false),
};
- let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
- .with_context("response", format!("{parts:?}"));
+ let message = String::from_utf8_lossy(&bs);
+
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/ghac/error.rs b/core/src/services/ghac/error.rs
index 005936b2d..598822f18 100644
--- a/core/src/services/ghac/error.rs
+++ b/core/src/services/ghac/error.rs
@@ -40,8 +40,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
};
let bs = body.bytes().await?;
- let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
- .with_context("response", format!("{parts:?}"));
+ let message = String::from_utf8_lossy(&bs);
+
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/http/error.rs b/core/src/services/http/error.rs
index aed2e13d6..c8afa09a9 100644
--- a/core/src/services/http/error.rs
+++ b/core/src/services/http/error.rs
@@ -41,8 +41,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
_ => (ErrorKind::Unexpected, false),
};
- let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
- .with_context("response", format!("{parts:?}"));
+ let message = String::from_utf8_lossy(&bs);
+
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/ipfs/error.rs b/core/src/services/ipfs/error.rs
index 7020dc43d..1470b2098 100644
--- a/core/src/services/ipfs/error.rs
+++ b/core/src/services/ipfs/error.rs
@@ -40,8 +40,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
_ => (ErrorKind::Unexpected, false),
};
- let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
- .with_context("response", format!("{parts:?}"));
+ let message = String::from_utf8_lossy(&bs);
+
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/ipmfs/error.rs b/core/src/services/ipmfs/error.rs
index 286eda36b..df6e7f475 100644
--- a/core/src/services/ipmfs/error.rs
+++ b/core/src/services/ipmfs/error.rs
@@ -73,7 +73,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
None => String::from_utf8_lossy(&bs).into_owned(),
};
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/obs/error.rs b/core/src/services/obs/error.rs
index ddcca5f39..0c2081312 100644
--- a/core/src/services/obs/error.rs
+++ b/core/src/services/obs/error.rs
@@ -63,7 +63,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
Err(_) => String::from_utf8_lossy(&bs).into_owned(),
};
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/onedrive/error.rs b/core/src/services/onedrive/error.rs
index 6fe6b57d4..60de6ae0d 100644
--- a/core/src/services/onedrive/error.rs
+++ b/core/src/services/onedrive/error.rs
@@ -38,8 +38,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
_ => (ErrorKind::Unexpected, false),
};
- let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
- .with_context("response", format!("{parts:?}"));
+ let message = String::from_utf8_lossy(&bs);
+
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/oss/error.rs b/core/src/services/oss/error.rs
index 715ad744f..ecead41c4 100644
--- a/core/src/services/oss/error.rs
+++ b/core/src/services/oss/error.rs
@@ -59,7 +59,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
Err(_) => String::from_utf8_lossy(&bs).into_owned(),
};
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/s3/error.rs b/core/src/services/s3/error.rs
index fce5c6d14..f68355487 100644
--- a/core/src/services/s3/error.rs
+++ b/core/src/services/s3/error.rs
@@ -17,7 +17,6 @@
use bytes::Buf;
use http::Response;
-use http::Uri;
use quick_xml::de;
use serde::Deserialize;
@@ -60,16 +59,14 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
(kind, retryable) = parse_s3_error_code(s3_err.code.as_str()).unwrap_or((kind, retryable));
}
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
}
- if let Some(uri) = parts.extensions.get::<Uri>() {
- err = err.with_context("uri", uri.to_string());
- }
-
Ok(err)
}
diff --git a/core/src/services/supabase/error.rs b/core/src/services/supabase/error.rs
index 41c4225b2..58288959b 100644
--- a/core/src/services/supabase/error.rs
+++ b/core/src/services/supabase/error.rs
@@ -53,7 +53,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
})
.unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/vercel_artifacts/error.rs b/core/src/services/vercel_artifacts/error.rs
index 6fe6b57d4..60de6ae0d 100644
--- a/core/src/services/vercel_artifacts/error.rs
+++ b/core/src/services/vercel_artifacts/error.rs
@@ -38,8 +38,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
_ => (ErrorKind::Unexpected, false),
};
- let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
- .with_context("response", format!("{parts:?}"));
+ let message = String::from_utf8_lossy(&bs);
+
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/wasabi/error.rs b/core/src/services/wasabi/error.rs
index 2e4423f48..74bb4aa2e 100644
--- a/core/src/services/wasabi/error.rs
+++ b/core/src/services/wasabi/error.rs
@@ -62,7 +62,9 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
(kind, retryable) = parse_wasabi_error_code(&wasabi_err.code).unwrap_or((kind, retryable));
}
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/webdav/error.rs b/core/src/services/webdav/error.rs
index 472e7e30a..967dc19f5 100644
--- a/core/src/services/webdav/error.rs
+++ b/core/src/services/webdav/error.rs
@@ -41,8 +41,11 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
_ => (ErrorKind::Unexpected, false),
};
- let mut err = Error::new(kind, &String::from_utf8_lossy(&bs))
- .with_context("response", format!("{parts:?}"));
+ let message = String::from_utf8_lossy(&bs);
+
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();
diff --git a/core/src/services/webhdfs/error.rs b/core/src/services/webhdfs/error.rs
index eafd36bee..7964ea383 100644
--- a/core/src/services/webhdfs/error.rs
+++ b/core/src/services/webhdfs/error.rs
@@ -65,7 +65,9 @@ fn parse_error_msg(parts: Parts, body: &str) -> Result<Error> {
Err(_) => body.to_owned(),
};
- let mut err = Error::new(kind, &message).with_context("response", format!("{parts:?}"));
+ let mut err = Error::new(kind, &message);
+
+ err = with_error_response_context(err, parts);
if retryable {
err = err.set_temporary();