You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/10/12 07:28:03 UTC
(iotdb) branch master updated: Pipe: omit region id when marking collect invocation count for remaining time calculation (#13673)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 357072543e4 Pipe: omit region id when marking collect invocation count for remaining time calculation (#13673)
357072543e4 is described below
commit 357072543e443d96af0fa9fa8ac99f6d4f6ed988
Author: V_Galaxy <vg...@apache.org>
AuthorDate: Sat Oct 12 15:27:57 2024 +0800
Pipe: omit region id when marking collect invocation count for remaining time calculation (#13673)
---
.../agent/task/stage/PipeTaskProcessorStage.java | 2 +-
.../task/subtask/processor/PipeProcessorSubtask.java | 20 ++++++++++++--------
.../agent/task/PipeProcessorSubtaskExecutorTest.java | 2 +-
.../agent/task/progress/PipeEventCommitManager.java | 3 +--
.../iotdb/commons/pipe/event/EnrichedEvent.java | 7 +++++++
5 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskProcessorStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskProcessorStage.java
index 7767792ce52..22cd5ce3094 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskProcessorStage.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskProcessorStage.java
@@ -104,8 +104,8 @@ public class PipeTaskProcessorStage extends PipeTaskStage {
this.pipeProcessorSubtask =
new PipeProcessorSubtask(
taskId,
- creationTime,
pipeName,
+ creationTime,
regionId,
pipeExtractorInputEventSupplier,
pipeProcessor,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java
index be7e23bed0b..2b44a34ae99 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java
@@ -57,33 +57,35 @@ public class PipeProcessorSubtask extends PipeReportableSubtask {
private static final AtomicReference<PipeProcessorSubtaskWorkerManager> subtaskWorkerManager =
new AtomicReference<>();
- private final EventSupplier inputEventSupplier;
- private final PipeProcessor pipeProcessor;
- private final PipeEventCollector outputEventCollector;
-
// Record these variables to provide corresponding value to tag key of monitoring metrics
private final String pipeName;
+ private final String pipeNameWithCreationTime; // cache for better performance
private final int regionId;
+ private final EventSupplier inputEventSupplier;
+ private final PipeProcessor pipeProcessor;
+ private final PipeEventCollector outputEventCollector;
+
// This variable is used to distinguish between old and new subtasks before and after stuck
// restart.
private final long subtaskCreationTime;
public PipeProcessorSubtask(
final String taskID,
- final long creationTime,
final String pipeName,
+ final long creationTime,
final int regionId,
final EventSupplier inputEventSupplier,
final PipeProcessor pipeProcessor,
final PipeEventCollector outputEventCollector) {
super(taskID, creationTime);
- this.subtaskCreationTime = System.currentTimeMillis();
this.pipeName = pipeName;
+ this.pipeNameWithCreationTime = pipeName + "_" + creationTime;
this.regionId = regionId;
this.inputEventSupplier = inputEventSupplier;
this.pipeProcessor = pipeProcessor;
this.outputEventCollector = outputEventCollector;
+ this.subtaskCreationTime = System.currentTimeMillis();
// Only register dataRegions
if (StorageEngine.getInstance().getAllDataRegionIds().contains(new DataRegionId(regionId))) {
@@ -137,12 +139,14 @@ public class PipeProcessorSubtask extends PipeReportableSubtask {
pipeProcessor.process((TabletInsertionEvent) event, outputEventCollector);
PipeProcessorMetrics.getInstance().markTabletEvent(taskID);
PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
- .markCollectInvocationCount(taskID, outputEventCollector.getCollectInvocationCount());
+ .markCollectInvocationCount(
+ pipeNameWithCreationTime, outputEventCollector.getCollectInvocationCount());
} else if (event instanceof TsFileInsertionEvent) {
pipeProcessor.process((TsFileInsertionEvent) event, outputEventCollector);
PipeProcessorMetrics.getInstance().markTsFileEvent(taskID);
PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
- .markCollectInvocationCount(taskID, outputEventCollector.getCollectInvocationCount());
+ .markCollectInvocationCount(
+ pipeNameWithCreationTime, outputEventCollector.getCollectInvocationCount());
} else if (event instanceof PipeHeartbeatEvent) {
pipeProcessor.process(event, outputEventCollector);
((PipeHeartbeatEvent) event).onProcessed();
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeProcessorSubtaskExecutorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeProcessorSubtaskExecutorTest.java
index 9b001f66e7f..4d3cf4c49d8 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeProcessorSubtaskExecutorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/PipeProcessorSubtaskExecutorTest.java
@@ -40,8 +40,8 @@ public class PipeProcessorSubtaskExecutorTest extends PipeSubtaskExecutorTest {
Mockito.spy(
new PipeProcessorSubtask(
"PipeProcessorSubtaskExecutorTest",
- System.currentTimeMillis(),
"TestPipe",
+ System.currentTimeMillis(),
0,
mock(EventSupplier.class),
mock(PipeProcessor.class),
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java
index 4d70679c6fe..c429f3155c0 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java
@@ -104,8 +104,7 @@ public class PipeEventCommitManager {
}
if (Objects.nonNull(commitRateMarker)) {
try {
- commitRateMarker.accept(
- event.getPipeName() + '_' + event.getCreationTime(), event.isDataRegionEvent());
+ commitRateMarker.accept(event.getPipeNameWithCreationTime(), event.isDataRegionEvent());
} catch (final Exception e) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java
index afed8d5f3bb..ee70ff5286c 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/EnrichedEvent.java
@@ -54,6 +54,8 @@ public abstract class EnrichedEvent implements Event {
protected final String pipeName;
protected final long creationTime;
+ private final String pipeNameWithCreationTime; // cache for better performance
+
protected final PipeTaskMeta pipeTaskMeta;
protected CommitterKey committerKey;
@@ -86,6 +88,7 @@ public abstract class EnrichedEvent implements Event {
this.pipeName = pipeName;
this.creationTime = creationTime;
+ this.pipeNameWithCreationTime = pipeName + "_" + creationTime;
this.pipeTaskMeta = pipeTaskMeta;
this.treePattern = treePattern;
this.tablePattern = tablePattern;
@@ -302,6 +305,10 @@ public abstract class EnrichedEvent implements Event {
return creationTime;
}
+ public String getPipeNameWithCreationTime() {
+ return pipeNameWithCreationTime;
+ }
+
public final int getRegionId() {
// TODO: persist regionId in EnrichedEvent
return committerKey == null ? -1 : committerKey.getRegionId();