You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2024/12/12 12:13:35 UTC

(iotdb) branch fix_threadpool_metric_npe updated (47e8fe4d917 -> 9501c0e3684)

This is an automated email from the ASF dual-hosted git repository.

tanxinyu pushed a change to branch fix_threadpool_metric_npe
in repository https://gitbox.apache.org/repos/asf/iotdb.git


 discard 47e8fe4d917 fix
    omit 46c97fb12b7 fix gile_global_count & dataregion mem leak
    omit 07f87996244 fix compile
    omit f31e1b014b1 fix threadpoolmetric leak & points asyncreporter metric leak
    omit bf43c22c86b fix concurrent bug
     add 21652cc2180 Fixed the bug of ID Like judgement in queries
     add 589866872db Fix group by nature month when timezone is not default
     add 70691e4b2be Add `removeInput` and `removable` to some specific Accmulators
     add ba824548e0b fix wal queue stuck and fix metric (#14387)
     add 37450ec160b Supprt trusted_uri_pattern in config file
     add e1884cad2f8 Support us and ns timePrecision in udf-library part-1
     add 839e2221dce Move model annotations to tsfile (#14358)
     add 9c65c326c4d Pipe: Deleted the "root" prefix dir in table model for OPC UA Sink (#14385)
     add 54197b95d28 Pipe: Fix the deadlock of PeriodicalJob thread caused by using parallelStream to split restartAllStuckPipes' subtasks (#14392)
     add d3b96d8136c show more use ctx.getIn(), because Sometimes it's not system. in (#14322)
     add 88ffa48e320 Fix ThreadPoolMetrics concurrent NPE bug & Fix metric leaks when frequently creating and deleting database (#14388)
     add 4be990922b4 [region migration] Remove retry of RegionMigrateService::addPeer (#14362)
     new 9501c0e3684 fix

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (47e8fe4d917)
            \
             N -- N -- N   refs/heads/fix_threadpool_metric_npe (9501c0e3684)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../org/apache/iotdb/TableModelSessionExample.java |   4 +-
 .../apache/iotdb/TableModelSessionPoolExample.java |   4 +-
 .../iotdb/it/env/cluster/env/AbstractEnv.java      |  15 ++
 .../iotdb/it/env/remote/env/RemoteServerEnv.java   |   9 ++
 .../java/org/apache/iotdb/itbase/env/BaseEnv.java  |   3 +
 .../org/apache/iotdb/ainode/it/AINodeBasicIT.java  |   1 +
 .../db/it/groupby/IoTDBGroupByNaturalMonthIT.java  |  28 ++++
 .../db/it/query/IoTDBEncryptionValueQueryIT.java   |   2 +
 .../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java      |  12 ++
 .../pipe/it/autocreate/IoTDBPipeLifeCycleIT.java   |   2 +-
 .../manual/IoTDBPipeTypeConversionISessionIT.java  |   3 +-
 .../it/db/it/udf/IoTDBSQLFunctionManagementIT.java |  16 +--
 .../it/db/it/udf/scalar/IoTDBScalarFunctionIT.java |  21 ++-
 .../iotdb/relational/it/schema/IoTDBDeviceIT.java  |  36 ++---
 .../java/org/apache/iotdb/cli/AbstractCli.java     |  14 +-
 .../org/apache/iotdb/cli/utils/JlineUtils.java     |  32 +++--
 .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4   |   2 +-
 .../apache/iotdb/consensus/iot/IoTConsensus.java   | 157 +++++++++++----------
 .../consensus/iot/IoTConsensusServerImpl.java      |  20 +--
 .../apache/iotdb/consensus/iot/StabilityTest.java  |  36 -----
 .../org/apache/iotdb/db/conf/IoTDBDescriptor.java  |  23 +++
 .../db/pipe/agent/task/PipeDataNodeTaskAgent.java  |   9 +-
 .../connector/protocol/opcua/OpcUaNameSpace.java   |  13 +-
 .../protocol/thrift/impl/ClientRPCServiceImpl.java |   4 +-
 .../timerangeiterator/AggrWindowIterator.java      |  26 ++--
 .../PreAggrWindowWithNaturalMonthIterator.java     |   8 +-
 .../TimeRangeIteratorFactory.java                  |   9 +-
 .../execution/operator/AggregationUtil.java        |   7 +-
 .../process/SlidingWindowAggregationOperator.java  |   7 +-
 .../relational/aggregation/AvgAccumulator.java     |  73 ++++++++++
 .../relational/aggregation/CountAccumulator.java   |  20 +++
 .../relational/aggregation/SumAccumulator.java     |  69 +++++++++
 .../relational/aggregation/TableAccumulator.java   |   8 ++
 .../aggregation/TableModeAccumulator.java          | 102 +++++++++++++
 .../aggregation/TableVarianceAccumulator.java      |  86 +++++++++++
 .../queryengine/plan/analyze/AnalyzeVisitor.java   |   4 +-
 .../plan/analyze/TemplatedAggregationAnalyze.java  |   4 +-
 .../execution/config/TableConfigTaskVisitor.java   |  20 ++-
 .../execution/config/TreeConfigTaskVisitor.java    |  42 +++++-
 .../config/executor/ClusterConfigTaskExecutor.java |  18 +--
 .../plan/optimization/LimitOffsetPushDown.java     |  20 ++-
 .../db/queryengine/plan/parser/ASTVisitor.java     |   4 +-
 .../plan/planner/OperatorTreeGenerator.java        |  31 ++--
 .../plan/statement/crud/InsertBaseStatement.java   |   2 +-
 .../crud/InsertMultiTabletsStatement.java          |   2 +-
 .../plan/statement/crud/InsertRowStatement.java    |   2 +-
 .../crud/InsertRowsOfOneDeviceStatement.java       |   2 +-
 .../plan/statement/crud/InsertRowsStatement.java   |   2 +-
 .../plan/statement/crud/LoadTsFileStatement.java   |   2 +-
 .../plan/statement/pipe/PipeEnrichedStatement.java |   3 +-
 .../iotdb/db/service/RegionMigrateService.java     |  49 +++----
 .../iotdb/db/service/metrics/WritingMetrics.java   |  12 +-
 .../dataregion/modification/ModEntry.java          |   2 +-
 .../wal/utils/MemoryControlledWALEntryQueue.java   |   2 +-
 .../db/storageengine/rescon/memory/SystemInfo.java |   2 +
 .../org/apache/iotdb/db/utils/DateTimeUtils.java   |   9 +-
 .../iotdb/db/utils/annotations/TableModel.java     |  33 -----
 .../iotdb/db/utils/annotations/TreeModel.java      |  33 -----
 .../schemaRegion/SchemaRegionTableDeviceTest.java  |  13 +-
 .../connector/PipeDataNodeThriftRequestTest.java   |  22 +--
 .../aggregation/TimeRangeIteratorTest.java         |  53 ++++---
 .../operator/AggregationOperatorTest.java          |   7 +-
 .../AlignedSeriesAggregationScanOperatorTest.java  |   3 +-
 .../operator/HorizontallyConcatOperatorTest.java   |   5 +-
 .../execution/operator/LastQueryOperatorTest.java  |   9 +-
 .../operator/LastQueryTreeSortOperatorTest.java    |   9 +-
 .../execution/operator/OperatorMemoryTest.java     |  16 ++-
 .../operator/RawDataAggregationOperatorTest.java   |   3 +-
 .../SeriesAggregationScanOperatorTest.java         |   3 +-
 .../SlidingWindowAggregationOperatorTest.java      |   8 +-
 .../operator/UpdateLastCacheOperatorTest.java      |   3 +-
 .../plan/optimization/LimitOffsetPushDownTest.java |  11 +-
 .../conf/iotdb-system.properties.template          |   6 +
 .../apache/iotdb/commons/conf/CommonConfig.java    |  11 ++
 .../commons/executable/ExecutableManager.java      |   5 +
 .../filter/impl/StringValueFilterVisitor.java      |   5 +-
 .../schema/filter/impl/values/LikeFilter.java      |   2 +-
 .../iotdb/library/dprofile/UDTFResample.java       |   6 +-
 .../iotdb/library/dquality/UDTFCompleteness.java   |   3 +-
 .../iotdb/library/dquality/UDTFConsistency.java    |   2 +-
 .../iotdb/library/dquality/UDTFTimeliness.java     |   3 +-
 .../iotdb/library/dquality/UDTFValidity.java       |   3 +-
 .../apache/iotdb/library/frequency/UDTFIFFT.java   |   7 +-
 .../library/series/UDTFConsecutiveSequences.java   |   5 +-
 .../library/series/UDTFConsecutiveWindows.java     |  11 +-
 .../java/org/apache/iotdb/library/util/Util.java   |  90 +++++++++---
 pom.xml                                            |   2 +-
 87 files changed, 1032 insertions(+), 475 deletions(-)
 delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/annotations/TableModel.java
 delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/annotations/TreeModel.java


(iotdb) 01/01: fix

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tanxinyu pushed a commit to branch fix_threadpool_metric_npe
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 9501c0e3684f9ba4af55a20fb7604f46b120d40f
Author: OneSizeFitQuorum <ta...@apache.org>
AuthorDate: Thu Dec 12 20:13:13 2024 +0800

    fix
    
    Signed-off-by: OneSizeFitQuorum <ta...@apache.org>
---
 .../commons/concurrent/ThreadPoolMetrics.java      | 139 ++++++++++++---------
 1 file changed, 77 insertions(+), 62 deletions(-)

diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadPoolMetrics.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadPoolMetrics.java
index 05b6c63b0a5..b5b5d5d09b0 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadPoolMetrics.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadPoolMetrics.java
@@ -48,87 +48,102 @@ public class ThreadPoolMetrics implements IMetricSet {
       notRegisteredPoolMap.put(name, pool);
     } else {
       registeredPoolMap.put(name, pool);
-      metricService.createAutoGauge(
-          SystemMetric.THREAD_POOL_ACTIVE_THREAD_COUNT.toString(),
-          MetricLevel.IMPORTANT,
-          registeredPoolMap,
-          map -> registeredPoolMap.get(name).getActiveCount(),
-          SystemTag.POOL_NAME.toString(),
-          name);
-      metricService.createAutoGauge(
-          SystemMetric.THREAD_POOL_CORE_SIZE.toString(),
-          MetricLevel.IMPORTANT,
-          registeredPoolMap,
-          map -> registeredPoolMap.get(name).getCorePoolSize(),
-          SystemTag.POOL_NAME.toString(),
-          name);
-      metricService.createAutoGauge(
-          SystemMetric.THREAD_POOL_WAITING_TASK_COUNT.toString(),
-          MetricLevel.IMPORTANT,
-          registeredPoolMap,
-          map -> registeredPoolMap.get(name).getQueueLength(),
-          SystemTag.POOL_NAME.toString(),
-          name);
-      metricService.createAutoGauge(
-          SystemMetric.THREAD_POOL_DONE_TASK_COUNT.toString(),
-          MetricLevel.IMPORTANT,
-          registeredPoolMap,
-          map -> registeredPoolMap.get(name).getCompletedTaskCount(),
-          SystemTag.POOL_NAME.toString(),
-          name);
-      metricService.createAutoGauge(
-          SystemMetric.THREAD_POOL_LARGEST_POOL_SIZE.toString(),
-          MetricLevel.IMPORTANT,
-          registeredPoolMap,
-          map -> registeredPoolMap.get(name).getLargestPoolSize(),
-          SystemTag.POOL_NAME.toString(),
-          name);
+      registerThreadPoolToMetric(name);
     }
   }
 
+  private void registerThreadPoolToMetric(String name) {
+    metricService.createAutoGauge(
+        SystemMetric.THREAD_POOL_ACTIVE_THREAD_COUNT.toString(),
+        MetricLevel.IMPORTANT,
+        registeredPoolMap,
+        map -> registeredPoolMap.get(name).getActiveCount(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+    metricService.createAutoGauge(
+        SystemMetric.THREAD_POOL_CORE_SIZE.toString(),
+        MetricLevel.IMPORTANT,
+        registeredPoolMap,
+        map -> registeredPoolMap.get(name).getCorePoolSize(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+    metricService.createAutoGauge(
+        SystemMetric.THREAD_POOL_WAITING_TASK_COUNT.toString(),
+        MetricLevel.IMPORTANT,
+        registeredPoolMap,
+        map -> registeredPoolMap.get(name).getQueueLength(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+    metricService.createAutoGauge(
+        SystemMetric.THREAD_POOL_DONE_TASK_COUNT.toString(),
+        MetricLevel.IMPORTANT,
+        registeredPoolMap,
+        map -> registeredPoolMap.get(name).getCompletedTaskCount(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+    metricService.createAutoGauge(
+        SystemMetric.THREAD_POOL_LARGEST_POOL_SIZE.toString(),
+        MetricLevel.IMPORTANT,
+        registeredPoolMap,
+        map -> registeredPoolMap.get(name).getLargestPoolSize(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+  }
+
   public synchronized void unRegisterThreadPool(String name) {
     if (metricService == null) {
       notRegisteredPoolMap.remove(name);
     } else {
       registeredPoolMap.remove(name);
-      metricService.remove(
-          MetricType.GAUGE,
-          SystemMetric.THREAD_POOL_ACTIVE_THREAD_COUNT.toString(),
-          SystemTag.POOL_NAME.toString(),
-          name);
-      metricService.remove(
-          MetricType.GAUGE,
-          SystemMetric.THREAD_POOL_CORE_SIZE.toString(),
-          SystemTag.POOL_NAME.toString(),
-          name);
-      metricService.remove(
-          MetricType.GAUGE,
-          SystemMetric.THREAD_POOL_WAITING_TASK_COUNT.toString(),
-          SystemTag.POOL_NAME.toString(),
-          name);
-      metricService.remove(
-          MetricType.GAUGE,
-          SystemMetric.THREAD_POOL_DONE_TASK_COUNT.toString(),
-          SystemTag.POOL_NAME.toString(),
-          name);
-      metricService.remove(
-          MetricType.GAUGE,
-          SystemMetric.THREAD_POOL_LARGEST_POOL_SIZE.toString(),
-          SystemTag.POOL_NAME.toString(),
-          name);
+      unRegisterThreadPoolToMetric(name);
     }
   }
 
+  private void unRegisterThreadPoolToMetric(String name) {
+    metricService.remove(
+        MetricType.GAUGE,
+        SystemMetric.THREAD_POOL_ACTIVE_THREAD_COUNT.toString(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+    metricService.remove(
+        MetricType.GAUGE,
+        SystemMetric.THREAD_POOL_CORE_SIZE.toString(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+    metricService.remove(
+        MetricType.GAUGE,
+        SystemMetric.THREAD_POOL_WAITING_TASK_COUNT.toString(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+    metricService.remove(
+        MetricType.GAUGE,
+        SystemMetric.THREAD_POOL_DONE_TASK_COUNT.toString(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+    metricService.remove(
+        MetricType.GAUGE,
+        SystemMetric.THREAD_POOL_LARGEST_POOL_SIZE.toString(),
+        SystemTag.POOL_NAME.toString(),
+        name);
+  }
+
   @Override
   public synchronized void bindTo(AbstractMetricService metricService) {
     this.metricService = metricService;
-    notRegisteredPoolMap.forEach((name, pool) -> registerThreadPool(pool, name));
+    for (Map.Entry<String, IThreadPoolMBean> entry : notRegisteredPoolMap.entrySet()) {
+      registerThreadPoolToMetric(entry.getKey());
+    }
+    registeredPoolMap.putAll(notRegisteredPoolMap);
     notRegisteredPoolMap.clear();
   }
 
   @Override
   public synchronized void unbindFrom(AbstractMetricService metricService) {
-    registeredPoolMap.forEach((name, pool) -> unRegisterThreadPool(name));
+    for (Map.Entry<String, IThreadPoolMBean> entry : registeredPoolMap.entrySet()) {
+      unRegisterThreadPoolToMetric(entry.getKey());
+    }
+    registeredPoolMap.clear();
+    notRegisteredPoolMap.clear();
   }
 
   private static class ThreadPoolMetricsHolder {