You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sh...@apache.org on 2026/08/31 10:00:17 UTC
(iotdb) 01/02: feat(metrics): record DataNode file system exceptions
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch diskExceptionCounter
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 66df54bcaf89ef86d4447a36bc6d15c334e365e5
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Mon Aug 31 12:26:21 2026 +0800
feat(metrics): record DataNode file system exceptions
---
.../service/metrics/DataNodeExceptionMetrics.java | 68 ++++++++++++++++++++++
.../db/service/metrics/DataNodeMetricsHelper.java | 1 +
.../db/storageengine/dataregion/DataRegion.java | 2 +
.../dataregion/memtable/TsFileProcessor.java | 3 +
.../dataregion/wal/buffer/WALBuffer.java | 2 +
.../wal/checkpoint/CheckpointManager.java | 2 +
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
7 files changed, 79 insertions(+)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java
new file mode 100644
index 00000000000..af2a867c339
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.service.metrics;
+
+import org.apache.iotdb.commons.service.metric.enums.Metric;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Counter;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import java.nio.file.FileSystemException;
+
+/** Records DataNode exceptions whose cause chain contains a {@link FileSystemException}. */
+public class DataNodeExceptionMetrics implements IMetricSet {
+
+ private static final DataNodeExceptionMetrics INSTANCE = new DataNodeExceptionMetrics();
+
+ private Counter fileSystemExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
+
+ private DataNodeExceptionMetrics() {
+ // singleton
+ }
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ fileSystemExceptionCounter =
+ metricService.getOrCreateCounter(
+ Metric.FILE_SYSTEM_EXCEPTION_COUNT.toString(), MetricLevel.IMPORTANT);
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ fileSystemExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
+ metricService.remove(MetricType.COUNTER, Metric.FILE_SYSTEM_EXCEPTION_COUNT.toString());
+ }
+
+ public void recordFileSystemException(Throwable throwable) {
+ for (Throwable current = throwable; current != null; current = current.getCause()) {
+ if (current instanceof FileSystemException) {
+ fileSystemExceptionCounter.inc();
+ return;
+ }
+ }
+ }
+
+ public static DataNodeExceptionMetrics getInstance() {
+ return INSTANCE;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
index e2204e8cf0b..9f6f4e57201 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
@@ -69,6 +69,7 @@ public class DataNodeMetricsHelper {
metricService.addMetricSet(CompactionMetrics.getInstance());
metricService.addMetricSet(new ProcessMetrics());
metricService.addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE));
+ metricService.addMetricSet(DataNodeExceptionMetrics.getInstance());
metricService.addMetricSet(new NetMetrics(IoTDBConstant.DN_ROLE));
metricService.addMetricSet(ClientManagerMetrics.getInstance());
metricService.addMetricSet(RPCServiceThriftHandlerMetrics.getInstance());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index cfdc63edbdf..fc06e7bc532 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -101,6 +101,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.Tr
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
import org.apache.iotdb.db.service.SettleService;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
@@ -2156,6 +2157,7 @@ public class DataRegion implements IDataRegionForQuery {
StorageEngineMessages
.STORAGE_LOG_MEET_IOEXCEPTION_WHEN_CREATING_TSFILEPROCESSOR_CHANGE_SYSTEM_4337F729,
e);
+ DataNodeExceptionMetrics.getInstance().recordFileSystemException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
throw new WriteProcessException(
String.format(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 759b58d4966..caed4732cb1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTablet
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.DataRegionInfo;
@@ -1867,6 +1868,7 @@ public class TsFileProcessor {
dataRegionName,
tsFileResource.getTsFile().getAbsolutePath(),
e);
+ DataNodeExceptionMetrics.getInstance().recordFileSystemException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
try {
logger.error(
@@ -1975,6 +1977,7 @@ public class TsFileProcessor {
logger.debug(StorageEngineMessages.FLUSHING_MEMTABLES_CLEAR, dataRegionName);
}
} catch (Exception e) {
+ DataNodeExceptionMetrics.getInstance().recordFileSystemException(e);
logger.error(
StorageEngineMessages.STORAGE_LOG_MARKING_OR_ENDING_FILE_MEET_ERROR_5653B904,
dataRegionName,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index 6921abecf75..097025e2e7d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointManager;
@@ -641,6 +642,7 @@ public class WALBuffer extends AbstractWALBuffer {
if (info.rollWALFileWriterListener != null) {
info.rollWALFileWriterListener.fail(e);
}
+ DataNodeExceptionMetrics.getInstance().recordFileSystemException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
}
} else if (forceFlag) { // force os cache to the storage device, avoid force twice by judging
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
index 1fc84166e05..a8245627763 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointWriter;
@@ -238,6 +239,7 @@ public class CheckpointManager implements AutoCloseable {
.STORAGE_LOG_FAIL_TO_ROLL_WAL_NODE_S_CHECKPOINT_WRITER_CHANGE_SYSTEM_791DDAB7,
identifier,
e);
+ DataNodeExceptionMetrics.getInstance().recordFileSystemException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
}
} finally {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index b41d8fb53ac..5e6eab5701b 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -94,6 +94,7 @@ public enum Metric {
PAM_ALLOCATED_FAILURE_COUNT("primitive_array_manager_allocated_failure_count"),
PAM_RELEASED_FAILURE_COUNT("primitive_array_manager_released_failure_count"),
TABLE_DISK_USAGE_CACHE("table_disk_usage_cache"),
+ FILE_SYSTEM_EXCEPTION_COUNT("file_system_exception_count"),
// compaction related
DATA_WRITTEN("data_written"),