You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/08/11 12:20:59 UTC
(iotdb) 19/47: Pipe: Exclude the tsFiles / insertNodes from transfer time metric which have not be sent (#16015)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e7333b5013ef34251b95a7650b1ef165f4dc97c1
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Wed Jul 23 16:55:40 2025 +0800
Pipe: Exclude the tsFiles / insertNodes from transfer time metric which have not be sent (#16015)
(cherry picked from commit f616e3441c089ad1728593da7d2b263fc39dbff0)
---
.../common/tablet/PipeInsertNodeTabletInsertionEvent.java | 5 ++++-
.../pipe/event/common/tsfile/PipeTsFileInsertionEvent.java | 5 ++++-
.../pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java | 12 ++++++++----
.../org/apache/iotdb/commons/pipe/event/EnrichedEvent.java | 6 +++---
4 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
index 329a123c489..ffa4da79466 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
@@ -211,7 +211,10 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
PipeDataNodeAgent.task()
.decreaseFloatingMemoryUsageInByte(pipeName, creationTime, ramBytesUsed());
PipeDataNodeSinglePipeMetrics.getInstance()
- .decreaseInsertNodeEventCount(pipeName, creationTime, System.nanoTime() - extractTime);
+ .decreaseInsertNodeEventCount(
+ pipeName,
+ creationTime,
+ shouldReportOnCommit ? System.nanoTime() - extractTime : -1);
}
insertNode = null;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index d791dddc713..ce9e70a2b42 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -337,7 +337,10 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
} finally {
if (Objects.nonNull(pipeName)) {
PipeDataNodeSinglePipeMetrics.getInstance()
- .decreaseTsFileEventCount(pipeName, creationTime, System.nanoTime() - extractTime);
+ .decreaseTsFileEventCount(
+ pipeName,
+ creationTime,
+ shouldReportOnCommit ? System.nanoTime() - extractTime : -1);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
index 65efae61714..70b4561bb5c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
@@ -269,8 +269,10 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime));
operator.decreaseInsertNodeEventCount();
- operator.getInsertNodeTransferTimer().update(transferTime, TimeUnit.NANOSECONDS);
- PIPE_DATANODE_INSERTNODE_TRANSFER_TIME_HISTOGRAM.update(transferTime);
+ if (transferTime > 0) {
+ operator.getInsertNodeTransferTimer().update(transferTime, TimeUnit.NANOSECONDS);
+ PIPE_DATANODE_INSERTNODE_TRANSFER_TIME_HISTOGRAM.update(transferTime);
+ }
}
public void increaseRawTabletEventCount(final String pipeName, final long creationTime) {
@@ -305,8 +307,10 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime));
operator.decreaseTsFileEventCount();
- operator.getTsFileTransferTimer().update(transferTime, TimeUnit.NANOSECONDS);
- PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM.update(transferTime);
+ if (transferTime > 0) {
+ operator.getTsFileTransferTimer().update(transferTime, TimeUnit.NANOSECONDS);
+ PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM.update(transferTime);
+ }
}
public void increaseHeartbeatEventCount(final String pipeName, final long creationTime) {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java
index 34e3f80dcf5..c7e0345cc59 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java
@@ -202,6 +202,9 @@ public abstract class EnrichedEvent implements Event {
}
if (referenceCount.get() == 1) {
+ if (!shouldReport) {
+ shouldReportOnCommit = false;
+ }
// We assume that this function will not throw any exceptions.
if (!internallyDecreaseResourceReferenceCount(holderMessage)) {
LOGGER.warn(
@@ -209,9 +212,6 @@ public abstract class EnrichedEvent implements Event {
coreReportMessage(),
Thread.currentThread().getStackTrace());
}
- if (!shouldReport) {
- shouldReportOnCommit = false;
- }
PipeEventCommitManager.getInstance().commit(this, committerKey);
}