You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by fj...@apache.org on 2019/05/10 05:30:51 UTC
[incubator-druid] branch master updated: Set direct memory if
unable to detect JVM config (#7606)
This is an automated email from the ASF dual-hosted git repository.
fjy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 1d49364 Set direct memory if unable to detect JVM config (#7606)
1d49364 is described below
commit 1d49364d08953cdd6730f2330a1f00908ed6b606
Author: Xavier Léauté <xl...@xvrl.net>
AuthorDate: Thu May 9 22:30:43 2019 -0700
Set direct memory if unable to detect JVM config (#7606)
* Set direct memory if unable to detect JVM config
Java 9 and above prevents us from detecting the maximum available direct
memory.
This change adds a fallback method to use at most 25% of maximum heap
size, which should be a reasonable default.
Unless -XX:MaxDirectMemorySize is set, recent JVMs will default maximum
direct memory to match the maximum heap size, so this should work out of
the box in most cases. For completeness we print instructions in the log
to explain how to adjust settings if necessary.
* skip test rather than succeeding
* reword log message
Co-Authored-By: Himanshu <g....@gmail.com>
---
.../apache/druid/query/DruidProcessingConfig.java | 28 ++++++++++++++++++----
.../apache/druid/guice/DruidProcessingModule.java | 9 +++++--
.../org/apache/druid/server/StatusResource.java | 10 +++++++-
.../druid/guice/DruidProcessingModuleTest.java | 11 +++++++++
4 files changed, 50 insertions(+), 8 deletions(-)
diff --git a/processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java b/processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java
index b35f297..c9afa1a 100644
--- a/processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java
+++ b/processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java
@@ -52,7 +52,22 @@ public abstract class DruidProcessingConfig extends ExecutorServiceConfig implem
return computedBufferSizeBytes.get();
}
- long directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
+ long directSizeBytes;
+ try {
+ directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
+ log.info(
+ "Detected max direct memory size of [%,d] bytes",
+ directSizeBytes
+ );
+ }
+ catch (UnsupportedOperationException e) {
+ // max direct memory defaults to max heap size on recent JDK version, unless set explicitly
+ directSizeBytes = computeMaxMemoryFromMaxHeapSize();
+ log.info(
+ "Defaulting to at most [%,d] bytes (25%% of max heap size) of direct memory for computation buffers",
+ directSizeBytes
+ );
+ }
int numProcessingThreads = getNumThreads();
int numMergeBuffers = getNumMergeBuffers();
@@ -62,17 +77,20 @@ public abstract class DruidProcessingConfig extends ExecutorServiceConfig implem
final int computedSizePerBuffer = Math.min(sizePerBuffer, MAX_DEFAULT_PROCESSING_BUFFER_SIZE_BYTES);
if (computedBufferSizeBytes.compareAndSet(null, computedSizePerBuffer)) {
log.info(
- "Auto sizing buffers to [%,d] bytes each for [%,d] processing and [%,d] merge buffers " +
- "out of [%,d] max direct memory",
+ "Auto sizing buffers to [%,d] bytes each for [%,d] processing and [%,d] merge buffers",
computedSizePerBuffer,
numProcessingThreads,
- numMergeBuffers,
- directSizeBytes
+ numMergeBuffers
);
}
return computedSizePerBuffer;
}
+ public static long computeMaxMemoryFromMaxHeapSize()
+ {
+ return Runtime.getRuntime().maxMemory() / 4;
+ }
+
@Config({"druid.computation.buffer.poolCacheMaxCount", "${base_path}.buffer.poolCacheMaxCount"})
public int poolCacheMaxCount()
{
diff --git a/server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java b/server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java
index fae4c10..efb1401 100644
--- a/server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java
+++ b/server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java
@@ -157,9 +157,14 @@ public class DruidProcessingModule implements Module
}
}
catch (UnsupportedOperationException e) {
+ log.debug("Checking for direct memory size is not support on this platform: %s", e);
log.info(
- "Could not verify that you have enough direct memory, so I hope you do! Error message was: %s",
- e.getMessage()
+ "Unable to determine max direct memory size. If druid.processing.buffer.sizeBytes is explicitly configured, "
+ + "then make sure to set -XX:MaxDirectMemorySize to at least \"druid.processing.buffer.sizeBytes * "
+ + "(druid.processing.numMergeBuffers[%,d] + druid.processing.numThreads[%,d] + 1)\", "
+ + "or else set -XX:MaxDirectMemorySize to at least 25%% of maximum jvm heap size.",
+ config.getNumMergeBuffers(),
+ config.getNumThreads()
);
}
}
diff --git a/server/src/main/java/org/apache/druid/server/StatusResource.java b/server/src/main/java/org/apache/druid/server/StatusResource.java
index 44b31ac..56fedc2 100644
--- a/server/src/main/java/org/apache/druid/server/StatusResource.java
+++ b/server/src/main/java/org/apache/druid/server/StatusResource.java
@@ -225,7 +225,15 @@ public class StatusResource
totalMemory = runtime.getTotalHeapSizeBytes();
freeMemory = runtime.getFreeHeapSizeBytes();
usedMemory = totalMemory - freeMemory;
- directMemory = runtime.getDirectMemorySizeBytes();
+
+ long directMemory = -1;
+ try {
+ directMemory = runtime.getDirectMemorySizeBytes();
+ }
+ catch (UnsupportedOperationException ignore) {
+ // querying direct memory is not supported
+ }
+ this.directMemory = directMemory;
}
@JsonProperty
diff --git a/server/src/test/java/org/apache/druid/guice/DruidProcessingModuleTest.java b/server/src/test/java/org/apache/druid/guice/DruidProcessingModuleTest.java
index add2d29..05f309d 100644
--- a/server/src/test/java/org/apache/druid/guice/DruidProcessingModuleTest.java
+++ b/server/src/test/java/org/apache/druid/guice/DruidProcessingModuleTest.java
@@ -21,6 +21,8 @@ package org.apache.druid.guice;
import com.google.inject.ProvisionException;
import org.apache.druid.query.DruidProcessingConfig;
+import org.apache.druid.utils.JvmUtils;
+import org.junit.Assume;
import org.junit.Test;
public class DruidProcessingModuleTest
@@ -29,6 +31,15 @@ public class DruidProcessingModuleTest
@Test(expected = ProvisionException.class)
public void testMemoryCheckThrowsException()
{
+ // JDK 9 and above do not support checking for direct memory size
+ // so this test only validates functionality for Java 8.
+ try {
+ JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
+ }
+ catch (UnsupportedOperationException e) {
+ Assume.assumeNoException(e);
+ }
+
DruidProcessingModule module = new DruidProcessingModule();
module.getIntermediateResultsPool(new DruidProcessingConfig()
{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org