You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/05/11 12:26:33 UTC

(stormcrawler) branch main updated: OkHttp protocol fails to cancel request after (#1900)

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git


The following commit(s) were added to refs/heads/main by this push:
     new 7a17307a OkHttp protocol fails to cancel request after (#1900)
7a17307a is described below

commit 7a17307a8f57d96215e96cc46de315ea1eb3125f
Author: Sebastian Nagel <se...@commoncrawl.org>
AuthorDate: Mon May 11 14:26:28 2026 +0200

    OkHttp protocol fails to cancel request after (#1900)
    
    topology.message.timeout.secs, fixes #1899
    
    - set call timeout to topology.message.timeout.secs (if not -1)
    - add type parameter to MutableObject
---
 .../stormcrawler/protocol/okhttp/HttpProtocol.java | 25 ++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
index 1a939c9c..cbcd4bf7 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
@@ -18,6 +18,7 @@
 package org.apache.stormcrawler.protocol.okhttp;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
 import java.net.Proxy;
@@ -160,6 +161,10 @@ public class HttpProtocol extends AbstractHttpProtocol {
                         .writeTimeout(timeout, TimeUnit.MILLISECONDS)
                         .readTimeout(timeout, TimeUnit.MILLISECONDS);
 
+        if (completionTimeout >= 0) {
+            builder.callTimeout(completionTimeout, TimeUnit.SECONDS);
+        }
+
         // protocols in order of preference, see
         // https://square.github.io/okhttp/4.x/okhttp/okhttp3/-ok-http-client/-builder/protocols/
         final List<okhttp3.Protocol> protocols = new ArrayList<>();
@@ -430,17 +435,18 @@ public class HttpProtocol extends AbstractHttpProtocol {
                 responsemetadata.addValue(key.toLowerCase(Locale.ROOT), value);
             }
 
-            final MutableObject trimmed = new MutableObject(TrimmedContentReason.NOT_TRIMMED);
+            final MutableObject<TrimmedContentReason> trimmed =
+                    new MutableObject<>(TrimmedContentReason.NOT_TRIMMED);
             final byte[] bytes = toByteArray(response.body(), pageMaxContent, trimmed);
-            if (trimmed.getValue() != TrimmedContentReason.NOT_TRIMMED) {
+            if (trimmed.get() != TrimmedContentReason.NOT_TRIMMED) {
                 if (!call.isCanceled()) {
                     call.cancel();
                 }
                 responsemetadata.setValue(ProtocolResponse.TRIMMED_RESPONSE_KEY, "true");
                 responsemetadata.setValue(
                         ProtocolResponse.TRIMMED_RESPONSE_REASON_KEY,
-                        trimmed.getValue().toString().toLowerCase(Locale.ROOT));
-                LOG.warn("HTTP content trimmed to {}", bytes.length);
+                        trimmed.get().toString().toLowerCase(Locale.ROOT));
+                LOG.warn("HTTP content trimmed to {} (reason: {})", bytes.length, trimmed.get());
             }
 
             final Long dnsResolution = DNStimes.remove(call.toString());
@@ -453,7 +459,9 @@ public class HttpProtocol extends AbstractHttpProtocol {
     }
 
     private byte[] toByteArray(
-            final ResponseBody responseBody, int maxContent, MutableObject trimmed)
+            final ResponseBody responseBody,
+            int maxContent,
+            MutableObject<TrimmedContentReason> trimmed)
             throws IOException {
 
         if (responseBody == null) {
@@ -493,7 +501,12 @@ public class HttpProtocol extends AbstractHttpProtocol {
                 // requesting more content failed, e.g. by a socket timeout
                 if (partialContentAsTrimmed && source.getBuffer().size() > 0) {
                     // treat already fetched content as trimmed
-                    trimmed.setValue(TrimmedContentReason.DISCONNECT);
+                    if (e instanceof InterruptedIOException) {
+                        // thrown by OkHttp if the call timeout is hit
+                        trimmed.setValue(TrimmedContentReason.TIME);
+                    } else {
+                        trimmed.setValue(TrimmedContentReason.DISCONNECT);
+                    }
                     LOG.debug("Exception while fetching {}", e);
                 } else {
                     throw e;