You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2022/08/03 00:57:33 UTC
[druid] branch master updated: Use nonzero default value of maxQueuedBytes. (#12840)
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2912a36a20 Use nonzero default value of maxQueuedBytes. (#12840)
2912a36a20 is described below
commit 2912a36a202672b2b025ef447c5bcbded7b20990
Author: Gian Merlino <gi...@gmail.com>
AuthorDate: Tue Aug 2 17:57:27 2022 -0700
Use nonzero default value of maxQueuedBytes. (#12840)
* Use nonzero default value of maxQueuedBytes.
The purpose of this parameter is to prevent the Broker from running out
of memory. The prior default is unlimited; this patch changes it to a
relatively conservative 25MB.
This may be too low for larger clusters. The risk is that throughput
can decrease for queries with large resultsets or large amounts of intermediate
data. However, I think this is better than the risk of the prior default, which
is that these queries can cause the Broker to go OOM.
* Alter calculation.
---
docs/configuration/index.md | 2 +-
.../apache/druid/guice/http/DruidHttpClientConfig.java | 16 ++++++++++++++--
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 2246727ad2..8f40989e48 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -1793,7 +1793,7 @@ client has the following configuration options.
|`druid.broker.http.compressionCodec`|Compression codec the Broker uses to communicate with Historical and real-time processes. May be "gzip" or "identity".|`gzip`|
|`druid.broker.http.readTimeout`|The timeout for data reads from Historical servers and real-time tasks.|`PT15M`|
|`druid.broker.http.unusedConnectionTimeout`|The timeout for idle connections in connection pool. The connection in the pool will be closed after this timeout and a new one will be established. This timeout should be less than `druid.broker.http.readTimeout`. Set this timeout = ~90% of `druid.broker.http.readTimeout`|`PT4M`|
-|`druid.broker.http.maxQueuedBytes`|Maximum number of bytes queued per query before exerting backpressure on the channel to the data server. Similar to `druid.server.http.maxScatterGatherBytes`, except unlike that configuration, this one will trigger backpressure rather than query failure. Zero means disabled. Can be overridden by the ["maxQueuedBytes" query context parameter](../querying/query-context.md). Human-readable format is supported, see [here](human-readable-byte.md). |`0` (disabled)|
+|`druid.broker.http.maxQueuedBytes`|Maximum number of bytes queued per query before exerting backpressure on channels to the data servers.<br /><br />Similar to `druid.server.http.maxScatterGatherBytes`, except unlike that configuration, this one will trigger backpressure rather than query failure. Zero means disabled. Can be overridden by the ["maxQueuedBytes" query context parameter](../querying/query-context.md). Human-readable format is supported, see [here](human-readable-byte.md). [...]
|`druid.broker.http.numMaxThreads`|`Maximum number of I/O worker threads|max(10, ((number of cores * 17) / 16 + 2) + 30)`|
##### Retry Policy
diff --git a/server/src/main/java/org/apache/druid/guice/http/DruidHttpClientConfig.java b/server/src/main/java/org/apache/druid/guice/http/DruidHttpClientConfig.java
index abc5cede85..d8e9db1385 100644
--- a/server/src/main/java/org/apache/druid/guice/http/DruidHttpClientConfig.java
+++ b/server/src/main/java/org/apache/druid/guice/http/DruidHttpClientConfig.java
@@ -29,11 +29,13 @@ import org.joda.time.Period;
import javax.validation.constraints.Min;
/**
+ *
*/
public class DruidHttpClientConfig
{
- private final String DEFAULT_COMPRESSION_CODEC = "gzip";
+ private static final String DEFAULT_COMPRESSION_CODEC = "gzip";
+ private static final double DEFAULT_MAX_QUEUED_BYTES_HEAP_FRACTION = 0.02; // Per query, so 2% is reasonably safe
private static final Logger LOG = new Logger(DruidHttpClientConfig.class);
@JsonProperty
@@ -65,7 +67,7 @@ public class DruidHttpClientConfig
* respected by CachingClusteredClient (broker -> data server communication).
*/
@JsonProperty
- private HumanReadableBytes maxQueuedBytes = HumanReadableBytes.ZERO;
+ private HumanReadableBytes maxQueuedBytes = computeDefaultMaxQueuedBytes();
@JsonProperty
private boolean eagerInitialization = true;
@@ -123,4 +125,14 @@ public class DruidHttpClientConfig
{
return eagerInitialization;
}
+
+ private static HumanReadableBytes computeDefaultMaxQueuedBytes()
+ {
+ return HumanReadableBytes.valueOf(
+ Math.max(
+ 25_000_000,
+ (long) (JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes() * DEFAULT_MAX_QUEUED_BYTES_HEAP_FRACTION)
+ )
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org