[GitHub] clintropolis commented on a change in pull request #6588: autosize processing buffers based on direct memory sizing by default

[GitBox <gi...@apache.org>]

clintropolis commented on a change in pull request #6588: autosize processing buffers based on direct memory sizing by default
URL: https://github.com/apache/incubator-druid/pull/6588#discussion_r232084240
 
 

 ##########
 File path: processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java
 ##########
 @@ -20,17 +20,58 @@
 package org.apache.druid.query;
 
 import org.apache.druid.java.util.common.concurrent.ExecutorServiceConfig;
+import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.segment.column.ColumnConfig;
+import org.apache.druid.utils.JvmUtils;
 import org.skife.config.Config;
 
+import java.util.concurrent.atomic.AtomicReference;
+
 public abstract class DruidProcessingConfig extends ExecutorServiceConfig implements ColumnConfig
 {
+  private static final Logger log = new Logger(DruidProcessingConfig.class);
+
   public static final int DEFAULT_NUM_MERGE_BUFFERS = -1;
+  public static final int DEFAULT_PROCESSING_BUFFER_SIZE_BYTES = -1;
+
+  private AtomicReference<Integer> computedBufferSizeBytes = new AtomicReference<>();
 
   @Config({"druid.computation.buffer.size", "${base_path}.buffer.sizeBytes"})
+  public int intermediateComputeSizeBytesConfigured()
+  {
+    return DEFAULT_PROCESSING_BUFFER_SIZE_BYTES;
+  }
+
   public int intermediateComputeSizeBytes()
   {
-    return 1024 * 1024 * 1024;
+    int sizeBytesConfigured = intermediateComputeSizeBytesConfigured();
+    if (sizeBytesConfigured != DEFAULT_PROCESSING_BUFFER_SIZE_BYTES) {
+      return sizeBytesConfigured;
+    } else if (computedBufferSizeBytes.get() != null) {
+      return computedBufferSizeBytes.get();
+    }
+
+    long directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
+
+    int numProcessingThreads = getNumThreads();
+    int numMergeBuffers = getNumMergeBuffers();
+    int totalNumBuffers = numMergeBuffers + numProcessingThreads;
+    int sizePerBuffer = (int) ((double) directSizeBytes / (double) (totalNumBuffers + 1));
+
+    // align down to nearest 64kb boundary
 
 Review comment:
   Oh, no good reason really, I was just playing around with slicing chunks off of each processing buffer to give to the spare space used by decompression buffers and forgot about this... it's probably not necessary, or at least with the current value, enough to be meaningful since each of those is 64k.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org