You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2024/12/12 12:13:35 UTC
(iotdb) branch fix_threadpool_metric_npe updated (47e8fe4d917 -> 9501c0e3684)
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a change to branch fix_threadpool_metric_npe
in repository https://gitbox.apache.org/repos/asf/iotdb.git
discard 47e8fe4d917 fix
omit 46c97fb12b7 fix gile_global_count & dataregion mem leak
omit 07f87996244 fix compile
omit f31e1b014b1 fix threadpoolmetric leak & points asyncreporter metric leak
omit bf43c22c86b fix concurrent bug
add 21652cc2180 Fixed the bug of ID Like judgement in queries
add 589866872db Fix group by nature month when timezone is not default
add 70691e4b2be Add `removeInput` and `removable` to some specific Accmulators
add ba824548e0b fix wal queue stuck and fix metric (#14387)
add 37450ec160b Supprt trusted_uri_pattern in config file
add e1884cad2f8 Support us and ns timePrecision in udf-library part-1
add 839e2221dce Move model annotations to tsfile (#14358)
add 9c65c326c4d Pipe: Deleted the "root" prefix dir in table model for OPC UA Sink (#14385)
add 54197b95d28 Pipe: Fix the deadlock of PeriodicalJob thread caused by using parallelStream to split restartAllStuckPipes' subtasks (#14392)
add d3b96d8136c show more use ctx.getIn(), because Sometimes it's not system. in (#14322)
add 88ffa48e320 Fix ThreadPoolMetrics concurrent NPE bug & Fix metric leaks when frequently creating and deleting database (#14388)
add 4be990922b4 [region migration] Remove retry of RegionMigrateService::addPeer (#14362)
new 9501c0e3684 fix
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (47e8fe4d917)
\
N -- N -- N refs/heads/fix_threadpool_metric_npe (9501c0e3684)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/TableModelSessionExample.java | 4 +-
.../apache/iotdb/TableModelSessionPoolExample.java | 4 +-
.../iotdb/it/env/cluster/env/AbstractEnv.java | 15 ++
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 9 ++
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 +
.../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 1 +
.../db/it/groupby/IoTDBGroupByNaturalMonthIT.java | 28 ++++
.../db/it/query/IoTDBEncryptionValueQueryIT.java | 2 +
.../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java | 12 ++
.../pipe/it/autocreate/IoTDBPipeLifeCycleIT.java | 2 +-
.../manual/IoTDBPipeTypeConversionISessionIT.java | 3 +-
.../it/db/it/udf/IoTDBSQLFunctionManagementIT.java | 16 +--
.../it/db/it/udf/scalar/IoTDBScalarFunctionIT.java | 21 ++-
.../iotdb/relational/it/schema/IoTDBDeviceIT.java | 36 ++---
.../java/org/apache/iotdb/cli/AbstractCli.java | 14 +-
.../org/apache/iotdb/cli/utils/JlineUtils.java | 32 +++--
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../apache/iotdb/consensus/iot/IoTConsensus.java | 157 +++++++++++----------
.../consensus/iot/IoTConsensusServerImpl.java | 20 +--
.../apache/iotdb/consensus/iot/StabilityTest.java | 36 -----
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 23 +++
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 9 +-
.../connector/protocol/opcua/OpcUaNameSpace.java | 13 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 4 +-
.../timerangeiterator/AggrWindowIterator.java | 26 ++--
.../PreAggrWindowWithNaturalMonthIterator.java | 8 +-
.../TimeRangeIteratorFactory.java | 9 +-
.../execution/operator/AggregationUtil.java | 7 +-
.../process/SlidingWindowAggregationOperator.java | 7 +-
.../relational/aggregation/AvgAccumulator.java | 73 ++++++++++
.../relational/aggregation/CountAccumulator.java | 20 +++
.../relational/aggregation/SumAccumulator.java | 69 +++++++++
.../relational/aggregation/TableAccumulator.java | 8 ++
.../aggregation/TableModeAccumulator.java | 102 +++++++++++++
.../aggregation/TableVarianceAccumulator.java | 86 +++++++++++
.../queryengine/plan/analyze/AnalyzeVisitor.java | 4 +-
.../plan/analyze/TemplatedAggregationAnalyze.java | 4 +-
.../execution/config/TableConfigTaskVisitor.java | 20 ++-
.../execution/config/TreeConfigTaskVisitor.java | 42 +++++-
.../config/executor/ClusterConfigTaskExecutor.java | 18 +--
.../plan/optimization/LimitOffsetPushDown.java | 20 ++-
.../db/queryengine/plan/parser/ASTVisitor.java | 4 +-
.../plan/planner/OperatorTreeGenerator.java | 31 ++--
.../plan/statement/crud/InsertBaseStatement.java | 2 +-
.../crud/InsertMultiTabletsStatement.java | 2 +-
.../plan/statement/crud/InsertRowStatement.java | 2 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 2 +-
.../plan/statement/crud/InsertRowsStatement.java | 2 +-
.../plan/statement/crud/LoadTsFileStatement.java | 2 +-
.../plan/statement/pipe/PipeEnrichedStatement.java | 3 +-
.../iotdb/db/service/RegionMigrateService.java | 49 +++----
.../iotdb/db/service/metrics/WritingMetrics.java | 12 +-
.../dataregion/modification/ModEntry.java | 2 +-
.../wal/utils/MemoryControlledWALEntryQueue.java | 2 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 2 +
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 9 +-
.../iotdb/db/utils/annotations/TableModel.java | 33 -----
.../iotdb/db/utils/annotations/TreeModel.java | 33 -----
.../schemaRegion/SchemaRegionTableDeviceTest.java | 13 +-
.../connector/PipeDataNodeThriftRequestTest.java | 22 +--
.../aggregation/TimeRangeIteratorTest.java | 53 ++++---
.../operator/AggregationOperatorTest.java | 7 +-
.../AlignedSeriesAggregationScanOperatorTest.java | 3 +-
.../operator/HorizontallyConcatOperatorTest.java | 5 +-
.../execution/operator/LastQueryOperatorTest.java | 9 +-
.../operator/LastQueryTreeSortOperatorTest.java | 9 +-
.../execution/operator/OperatorMemoryTest.java | 16 ++-
.../operator/RawDataAggregationOperatorTest.java | 3 +-
.../SeriesAggregationScanOperatorTest.java | 3 +-
.../SlidingWindowAggregationOperatorTest.java | 8 +-
.../operator/UpdateLastCacheOperatorTest.java | 3 +-
.../plan/optimization/LimitOffsetPushDownTest.java | 11 +-
.../conf/iotdb-system.properties.template | 6 +
.../apache/iotdb/commons/conf/CommonConfig.java | 11 ++
.../commons/executable/ExecutableManager.java | 5 +
.../filter/impl/StringValueFilterVisitor.java | 5 +-
.../schema/filter/impl/values/LikeFilter.java | 2 +-
.../iotdb/library/dprofile/UDTFResample.java | 6 +-
.../iotdb/library/dquality/UDTFCompleteness.java | 3 +-
.../iotdb/library/dquality/UDTFConsistency.java | 2 +-
.../iotdb/library/dquality/UDTFTimeliness.java | 3 +-
.../iotdb/library/dquality/UDTFValidity.java | 3 +-
.../apache/iotdb/library/frequency/UDTFIFFT.java | 7 +-
.../library/series/UDTFConsecutiveSequences.java | 5 +-
.../library/series/UDTFConsecutiveWindows.java | 11 +-
.../java/org/apache/iotdb/library/util/Util.java | 90 +++++++++---
pom.xml | 2 +-
87 files changed, 1032 insertions(+), 475 deletions(-)
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/annotations/TableModel.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/annotations/TreeModel.java
(iotdb) 01/01: fix
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch fix_threadpool_metric_npe
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9501c0e3684f9ba4af55a20fb7604f46b120d40f
Author: OneSizeFitQuorum <ta...@apache.org>
AuthorDate: Thu Dec 12 20:13:13 2024 +0800
fix
Signed-off-by: OneSizeFitQuorum <ta...@apache.org>
---
.../commons/concurrent/ThreadPoolMetrics.java | 139 ++++++++++++---------
1 file changed, 77 insertions(+), 62 deletions(-)
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadPoolMetrics.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadPoolMetrics.java
index 05b6c63b0a5..b5b5d5d09b0 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadPoolMetrics.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadPoolMetrics.java
@@ -48,87 +48,102 @@ public class ThreadPoolMetrics implements IMetricSet {
notRegisteredPoolMap.put(name, pool);
} else {
registeredPoolMap.put(name, pool);
- metricService.createAutoGauge(
- SystemMetric.THREAD_POOL_ACTIVE_THREAD_COUNT.toString(),
- MetricLevel.IMPORTANT,
- registeredPoolMap,
- map -> registeredPoolMap.get(name).getActiveCount(),
- SystemTag.POOL_NAME.toString(),
- name);
- metricService.createAutoGauge(
- SystemMetric.THREAD_POOL_CORE_SIZE.toString(),
- MetricLevel.IMPORTANT,
- registeredPoolMap,
- map -> registeredPoolMap.get(name).getCorePoolSize(),
- SystemTag.POOL_NAME.toString(),
- name);
- metricService.createAutoGauge(
- SystemMetric.THREAD_POOL_WAITING_TASK_COUNT.toString(),
- MetricLevel.IMPORTANT,
- registeredPoolMap,
- map -> registeredPoolMap.get(name).getQueueLength(),
- SystemTag.POOL_NAME.toString(),
- name);
- metricService.createAutoGauge(
- SystemMetric.THREAD_POOL_DONE_TASK_COUNT.toString(),
- MetricLevel.IMPORTANT,
- registeredPoolMap,
- map -> registeredPoolMap.get(name).getCompletedTaskCount(),
- SystemTag.POOL_NAME.toString(),
- name);
- metricService.createAutoGauge(
- SystemMetric.THREAD_POOL_LARGEST_POOL_SIZE.toString(),
- MetricLevel.IMPORTANT,
- registeredPoolMap,
- map -> registeredPoolMap.get(name).getLargestPoolSize(),
- SystemTag.POOL_NAME.toString(),
- name);
+ registerThreadPoolToMetric(name);
}
}
+ private void registerThreadPoolToMetric(String name) {
+ metricService.createAutoGauge(
+ SystemMetric.THREAD_POOL_ACTIVE_THREAD_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ registeredPoolMap,
+ map -> registeredPoolMap.get(name).getActiveCount(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ metricService.createAutoGauge(
+ SystemMetric.THREAD_POOL_CORE_SIZE.toString(),
+ MetricLevel.IMPORTANT,
+ registeredPoolMap,
+ map -> registeredPoolMap.get(name).getCorePoolSize(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ metricService.createAutoGauge(
+ SystemMetric.THREAD_POOL_WAITING_TASK_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ registeredPoolMap,
+ map -> registeredPoolMap.get(name).getQueueLength(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ metricService.createAutoGauge(
+ SystemMetric.THREAD_POOL_DONE_TASK_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ registeredPoolMap,
+ map -> registeredPoolMap.get(name).getCompletedTaskCount(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ metricService.createAutoGauge(
+ SystemMetric.THREAD_POOL_LARGEST_POOL_SIZE.toString(),
+ MetricLevel.IMPORTANT,
+ registeredPoolMap,
+ map -> registeredPoolMap.get(name).getLargestPoolSize(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ }
+
public synchronized void unRegisterThreadPool(String name) {
if (metricService == null) {
notRegisteredPoolMap.remove(name);
} else {
registeredPoolMap.remove(name);
- metricService.remove(
- MetricType.GAUGE,
- SystemMetric.THREAD_POOL_ACTIVE_THREAD_COUNT.toString(),
- SystemTag.POOL_NAME.toString(),
- name);
- metricService.remove(
- MetricType.GAUGE,
- SystemMetric.THREAD_POOL_CORE_SIZE.toString(),
- SystemTag.POOL_NAME.toString(),
- name);
- metricService.remove(
- MetricType.GAUGE,
- SystemMetric.THREAD_POOL_WAITING_TASK_COUNT.toString(),
- SystemTag.POOL_NAME.toString(),
- name);
- metricService.remove(
- MetricType.GAUGE,
- SystemMetric.THREAD_POOL_DONE_TASK_COUNT.toString(),
- SystemTag.POOL_NAME.toString(),
- name);
- metricService.remove(
- MetricType.GAUGE,
- SystemMetric.THREAD_POOL_LARGEST_POOL_SIZE.toString(),
- SystemTag.POOL_NAME.toString(),
- name);
+ unRegisterThreadPoolToMetric(name);
}
}
+ private void unRegisterThreadPoolToMetric(String name) {
+ metricService.remove(
+ MetricType.GAUGE,
+ SystemMetric.THREAD_POOL_ACTIVE_THREAD_COUNT.toString(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ metricService.remove(
+ MetricType.GAUGE,
+ SystemMetric.THREAD_POOL_CORE_SIZE.toString(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ metricService.remove(
+ MetricType.GAUGE,
+ SystemMetric.THREAD_POOL_WAITING_TASK_COUNT.toString(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ metricService.remove(
+ MetricType.GAUGE,
+ SystemMetric.THREAD_POOL_DONE_TASK_COUNT.toString(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ metricService.remove(
+ MetricType.GAUGE,
+ SystemMetric.THREAD_POOL_LARGEST_POOL_SIZE.toString(),
+ SystemTag.POOL_NAME.toString(),
+ name);
+ }
+
@Override
public synchronized void bindTo(AbstractMetricService metricService) {
this.metricService = metricService;
- notRegisteredPoolMap.forEach((name, pool) -> registerThreadPool(pool, name));
+ for (Map.Entry<String, IThreadPoolMBean> entry : notRegisteredPoolMap.entrySet()) {
+ registerThreadPoolToMetric(entry.getKey());
+ }
+ registeredPoolMap.putAll(notRegisteredPoolMap);
notRegisteredPoolMap.clear();
}
@Override
public synchronized void unbindFrom(AbstractMetricService metricService) {
- registeredPoolMap.forEach((name, pool) -> unRegisterThreadPool(name));
+ for (Map.Entry<String, IThreadPoolMBean> entry : registeredPoolMap.entrySet()) {
+ unRegisterThreadPoolToMetric(entry.getKey());
+ }
+ registeredPoolMap.clear();
+ notRegisteredPoolMap.clear();
}
private static class ThreadPoolMetricsHolder {