[nutch] branch 2.x updated: NUTCH-2548 Compressed content skipped, contributed by Rustam - do not store content length from HTTP header if content was compressed

[sn...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

snagel pushed a commit to branch 2.x
in repository https://gitbox.apache.org/repos/asf/nutch.git


The following commit(s) were added to refs/heads/2.x by this push:
     new 7f0fe0f  NUTCH-2548 Compressed content skipped, contributed by Rustam - do not store content length from HTTP header if content was compressed
7f0fe0f is described below

commit 7f0fe0fc718cf1caf4bb2ad3c0d4d2e01d92e571
Author: Sebastian Nagel <sn...@apache.org>
AuthorDate: Mon Apr 9 11:50:13 2018 +0200

    NUTCH-2548 Compressed content skipped, contributed by Rustam
    - do not store content length from HTTP header if content was compressed
---
 .../src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
index 989e4e5..3140bee 100644
--- a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
+++ b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
@@ -169,10 +169,12 @@ public class HttpResponse implements Response {
           content = http.processGzipEncoded(content, url);
           if (Http.LOG.isTraceEnabled())
             fetchTrace.append("; extracted to " + content.length + " bytes");
+          headers.remove(Response.CONTENT_LENGTH);
         } else if ("deflate".equals(contentEncoding)) {
           content = http.processDeflateEncoded(content, url);
           if (Http.LOG.isTraceEnabled())
             fetchTrace.append("; extracted to " + content.length + " bytes");
+          headers.remove(Response.CONTENT_LENGTH);
         }
       }
 

-- 
To stop receiving notification emails like this one, please contact
snagel@apache.org.