[GitHub] QiuMM commented on a change in pull request #6588: autosize processing buffers based on direct memory sizing by default

[GitBox <gi...@apache.org>]

QiuMM commented on a change in pull request #6588: autosize processing buffers based on direct memory sizing by default
URL: https://github.com/apache/incubator-druid/pull/6588#discussion_r231989964
 
 

 ##########
 File path: processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java
 ##########
 @@ -20,17 +20,58 @@
 package org.apache.druid.query;
 
 import org.apache.druid.java.util.common.concurrent.ExecutorServiceConfig;
+import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.segment.column.ColumnConfig;
+import org.apache.druid.utils.JvmUtils;
 import org.skife.config.Config;
 
+import java.util.concurrent.atomic.AtomicReference;
+
 public abstract class DruidProcessingConfig extends ExecutorServiceConfig implements ColumnConfig
 {
+  private static final Logger log = new Logger(DruidProcessingConfig.class);
+
   public static final int DEFAULT_NUM_MERGE_BUFFERS = -1;
+  public static final int DEFAULT_PROCESSING_BUFFER_SIZE_BYTES = -1;
+
+  private AtomicReference<Integer> computedBufferSizeBytes = new AtomicReference<>();
 
   @Config({"druid.computation.buffer.size", "${base_path}.buffer.sizeBytes"})
+  public int intermediateComputeSizeBytesConfigured()
+  {
+    return DEFAULT_PROCESSING_BUFFER_SIZE_BYTES;
+  }
+
   public int intermediateComputeSizeBytes()
   {
-    return 1024 * 1024 * 1024;
+    int sizeBytesConfigured = intermediateComputeSizeBytesConfigured();
+    if (sizeBytesConfigured != DEFAULT_PROCESSING_BUFFER_SIZE_BYTES) {
+      return sizeBytesConfigured;
+    } else if (computedBufferSizeBytes.get() != null) {
+      return computedBufferSizeBytes.get();
+    }
+
+    long directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
+
+    int numProcessingThreads = getNumThreads();
+    int numMergeBuffers = getNumMergeBuffers();
+    int totalNumBuffers = numMergeBuffers + numProcessingThreads;
+    int sizePerBuffer = (int) ((double) directSizeBytes / (double) (totalNumBuffers + 1));
 
 Review comment:
   Just `(int) ((double) directSizeBytes /  (totalNumBuffers + 1));` would be ok.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org