[GitHub] [incubator-druid] asdf2014 commented on a change in pull request #8201: Simple memory allocation for CliIndexer tasks

[GitBox <gi...@apache.org>]

asdf2014 commented on a change in pull request #8201: Simple memory allocation for CliIndexer tasks
URL: https://github.com/apache/incubator-druid/pull/8201#discussion_r309026818
 
 

 ##########
 File path: server/src/main/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManager.java
 ##########
 @@ -21,62 +21,102 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
 import com.google.inject.Inject;
 import org.apache.druid.client.cache.Cache;
 import org.apache.druid.client.cache.CacheConfig;
 import org.apache.druid.client.cache.CachePopulatorStats;
 import org.apache.druid.guice.annotations.Processing;
+import org.apache.druid.indexing.worker.config.WorkerConfig;
 import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.UOE;
+import org.apache.druid.java.util.common.concurrent.Execs;
+import org.apache.druid.java.util.emitter.EmittingLogger;
 import org.apache.druid.java.util.emitter.service.ServiceEmitter;
 import org.apache.druid.query.Query;
 import org.apache.druid.query.QueryRunner;
 import org.apache.druid.query.QueryRunnerFactoryConglomerate;
 import org.apache.druid.query.SegmentDescriptor;
+import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.IndexMerger;
+import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.IndexableAdapter;
+import org.apache.druid.segment.ProgressIndicator;
+import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.incremental.IncrementalIndex;
 import org.apache.druid.segment.indexing.DataSchema;
 import org.apache.druid.segment.loading.DataSegmentPusher;
 import org.apache.druid.segment.realtime.FireDepartmentMetrics;
 import org.apache.druid.segment.realtime.plumber.Sink;
+import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory;
 import org.apache.druid.server.coordination.DataSegmentAnnouncer;
 import org.apache.druid.timeline.VersionedIntervalTimeline;
 import org.joda.time.Interval;
+import org.joda.time.Period;
 
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 
 /**
- * Manages Appenderators for the Indexer task execution service, which runs all tasks in a single process.
+ * Manages {@link Appenderator} instances for the Indexer task execution service, which runs all tasks in
+ * a single process.
  *
  * This class keeps two maps:
- * - A per-datasource SinkQuerySegmentWalker (with an associated per-datasource timeline)
+ * - A per-datasource {@link SinkQuerySegmentWalker} (with an associated per-datasource timeline)
  * - A map that associates a taskId with the Appenderator created for that task
  *
  * Appenderators created by this class will use the shared per-datasource SinkQuerySegmentWalkers.
  *
  * The per-datasource SinkQuerySegmentWalkers share a common queryExecutorService.
+ *
+ * Each task that requests an Appenderator from this AppenderatorsManager will receive a heap memory limit
+ * equal to `druid.worker.globalIngestionHeapLimitBytes` evenly divided by `druid.worker.capacity`.
+ *
+ * The Appenderators created by this class share an executor pool for {@link IndexMerger} persist
+ * and merge operations, with concurrent operations limited to `druid.worker.capacity` divided 2. This limit is imposed
+ * to reduce overall memory usage.
  */
 public class UnifiedIndexerAppenderatorsManager implements AppenderatorsManager
 {
+  private static final EmittingLogger LOG = new EmittingLogger(UnifiedIndexerAppenderatorsManager.class);
 
 Review comment:
   The `EmittingLogger LOG` is unnecessary

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org