(incubator-opendal) branch fix-content-encoding created (now 94e9a9f79c)

[xu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a change to branch fix-content-encoding
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


      at 94e9a9f79c fix(core): Handling content encoding correctly

This branch includes the following new commits:

     new 94e9a9f79c fix(core): Handling content encoding correctly

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(incubator-opendal) 01/01: fix(core): Handling content encoding correctly

[xu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch fix-content-encoding
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit 94e9a9f79c33f8cb03440fcedfc89f9d07b6672d
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Jan 4 14:21:02 2024 +0800

    fix(core): Handling content encoding correctly
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/src/raw/http_util/client.rs | 9 ++++++---
 core/src/raw/http_util/header.rs | 7 ++++++-
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/core/src/raw/http_util/client.rs b/core/src/raw/http_util/client.rs
index 3bd7e826cd..94cbd1dd39 100644
--- a/core/src/raw/http_util/client.rs
+++ b/core/src/raw/http_util/client.rs
@@ -28,6 +28,7 @@ use http::Response;
 use super::body::IncomingAsyncBody;
 use super::parse_content_length;
 use super::AsyncBody;
+use crate::raw::http_util::header::parse_content_encoding;
 use crate::raw::*;
 use crate::Error;
 use crate::ErrorKind;
@@ -167,11 +168,13 @@ impl HttpClient {
         })?;
 
         // Get content length from header so that we can check it.
-        // If the request method is HEAD, we will ignore this.
-        let content_length = if is_head {
+        //
+        // - If the request method is HEAD, we will ignore content length.
+        // - If response contains content_encoding, we should omit it's content length.
+        let content_length = if is_head || parse_content_encoding(resp.headers())?.is_some() {
             None
         } else {
-            parse_content_length(resp.headers()).expect("response content length must be valid")
+            parse_content_length(resp.headers())?
         };
 
         let mut hr = Response::builder()
diff --git a/core/src/raw/http_util/header.rs b/core/src/raw/http_util/header.rs
index f266f7c0c4..ec28113a70 100644
--- a/core/src/raw/http_util/header.rs
+++ b/core/src/raw/http_util/header.rs
@@ -19,7 +19,6 @@ use base64::engine::general_purpose;
 use base64::Engine;
 use chrono::DateTime;
 use chrono::Utc;
-use http::header::CACHE_CONTROL;
 use http::header::CONTENT_DISPOSITION;
 use http::header::CONTENT_LENGTH;
 use http::header::CONTENT_RANGE;
@@ -27,6 +26,7 @@ use http::header::CONTENT_TYPE;
 use http::header::ETAG;
 use http::header::LAST_MODIFIED;
 use http::header::LOCATION;
+use http::header::{CACHE_CONTROL, CONTENT_ENCODING};
 use http::HeaderValue;
 use http::{HeaderMap, HeaderName};
 use md5::Digest;
@@ -77,6 +77,11 @@ pub fn parse_content_type(headers: &HeaderMap) -> Result<Option<&str>> {
     parse_header_to_str(headers, CONTENT_TYPE)
 }
 
+/// Parse content encoding from header map.
+pub fn parse_content_encoding(headers: &HeaderMap) -> Result<Option<&str>> {
+    parse_header_to_str(headers, CONTENT_ENCODING)
+}
+
 /// Parse content range from header map.
 pub fn parse_content_range(headers: &HeaderMap) -> Result<Option<BytesContentRange>> {
     parse_header_to_str(headers, CONTENT_RANGE)?