You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/05/11 12:26:33 UTC
(stormcrawler) branch main updated: OkHttp protocol fails to cancel request after (#1900)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new 7a17307a OkHttp protocol fails to cancel request after (#1900)
7a17307a is described below
commit 7a17307a8f57d96215e96cc46de315ea1eb3125f
Author: Sebastian Nagel <se...@commoncrawl.org>
AuthorDate: Mon May 11 14:26:28 2026 +0200
OkHttp protocol fails to cancel request after (#1900)
topology.message.timeout.secs, fixes #1899
- set call timeout to topology.message.timeout.secs (if not -1)
- add type parameter to MutableObject
---
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 25 ++++++++++++++++------
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
index 1a939c9c..cbcd4bf7 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
@@ -18,6 +18,7 @@
package org.apache.stormcrawler.protocol.okhttp;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
@@ -160,6 +161,10 @@ public class HttpProtocol extends AbstractHttpProtocol {
.writeTimeout(timeout, TimeUnit.MILLISECONDS)
.readTimeout(timeout, TimeUnit.MILLISECONDS);
+ if (completionTimeout >= 0) {
+ builder.callTimeout(completionTimeout, TimeUnit.SECONDS);
+ }
+
// protocols in order of preference, see
// https://square.github.io/okhttp/4.x/okhttp/okhttp3/-ok-http-client/-builder/protocols/
final List<okhttp3.Protocol> protocols = new ArrayList<>();
@@ -430,17 +435,18 @@ public class HttpProtocol extends AbstractHttpProtocol {
responsemetadata.addValue(key.toLowerCase(Locale.ROOT), value);
}
- final MutableObject trimmed = new MutableObject(TrimmedContentReason.NOT_TRIMMED);
+ final MutableObject<TrimmedContentReason> trimmed =
+ new MutableObject<>(TrimmedContentReason.NOT_TRIMMED);
final byte[] bytes = toByteArray(response.body(), pageMaxContent, trimmed);
- if (trimmed.getValue() != TrimmedContentReason.NOT_TRIMMED) {
+ if (trimmed.get() != TrimmedContentReason.NOT_TRIMMED) {
if (!call.isCanceled()) {
call.cancel();
}
responsemetadata.setValue(ProtocolResponse.TRIMMED_RESPONSE_KEY, "true");
responsemetadata.setValue(
ProtocolResponse.TRIMMED_RESPONSE_REASON_KEY,
- trimmed.getValue().toString().toLowerCase(Locale.ROOT));
- LOG.warn("HTTP content trimmed to {}", bytes.length);
+ trimmed.get().toString().toLowerCase(Locale.ROOT));
+ LOG.warn("HTTP content trimmed to {} (reason: {})", bytes.length, trimmed.get());
}
final Long dnsResolution = DNStimes.remove(call.toString());
@@ -453,7 +459,9 @@ public class HttpProtocol extends AbstractHttpProtocol {
}
private byte[] toByteArray(
- final ResponseBody responseBody, int maxContent, MutableObject trimmed)
+ final ResponseBody responseBody,
+ int maxContent,
+ MutableObject<TrimmedContentReason> trimmed)
throws IOException {
if (responseBody == null) {
@@ -493,7 +501,12 @@ public class HttpProtocol extends AbstractHttpProtocol {
// requesting more content failed, e.g. by a socket timeout
if (partialContentAsTrimmed && source.getBuffer().size() > 0) {
// treat already fetched content as trimmed
- trimmed.setValue(TrimmedContentReason.DISCONNECT);
+ if (e instanceof InterruptedIOException) {
+ // thrown by OkHttp if the call timeout is hit
+ trimmed.setValue(TrimmedContentReason.TIME);
+ } else {
+ trimmed.setValue(TrimmedContentReason.DISCONNECT);
+ }
LOG.debug("Exception while fetching {}", e);
} else {
throw e;