You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ju...@apache.org on 2026/06/29 10:35:12 UTC
(iotdb) 01/01: Fix stale pipe runtime error messages
This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch fix/clear-pipe-runtime-error-message
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4227fb55b3a34bab1fc89d89e0ae1dbb52ffd659
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Mon Jun 29 18:34:33 2026 +0800
Fix stale pipe runtime error messages
---
.../response/pipe/task/PipeTableResp.java | 6 +
.../runtime/heartbeat/PipeHeartbeatParser.java | 19 ++-
.../confignode/persistence/pipe/PipeTaskInfo.java | 33 ++++--
.../impl/pipe/task/StartPipeProcedureV2.java | 37 +++++-
.../runtime/heartbeat/PipeHeartbeatParserTest.java | 129 ++++++++++++++++++++-
.../pipe/PipeTaskInfoAutoRestartTest.java | 28 +++++
.../commons/pipe/agent/task/PipeTaskAgent.java | 13 +++
.../pipe/agent/task/meta/PipeRuntimeMeta.java | 9 ++
.../commons/pipe/agent/task/meta/PipeTaskMeta.java | 4 +
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 26 +++++
10 files changed, 283 insertions(+), 21 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java
index 49f10a79e8a..a2d2856e751 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java
@@ -194,6 +194,9 @@ public class PipeTableResp implements DataSet {
runtimeMeta.getNodeId2PipeRuntimeExceptionMap().entrySet()) {
final Integer nodeId = entry.getKey();
final PipeRuntimeException e = entry.getValue();
+ if (e.getTimeStamp() <= runtimeMeta.getExceptionsClearTime()) {
+ continue;
+ }
final String exceptionMessage =
DateTimeUtils.convertLongToDate(e.getTimeStamp(), "ms") + ", " + e.getMessage();
@@ -206,6 +209,9 @@ public class PipeTableResp implements DataSet {
runtimeMeta.getConsensusGroupId2TaskMetaMap().entrySet()) {
final Integer regionId = entry.getKey();
for (final PipeRuntimeException e : entry.getValue().getExceptionMessages()) {
+ if (e.getTimeStamp() <= runtimeMeta.getExceptionsClearTime()) {
+ continue;
+ }
final String exceptionMessage =
DateTimeUtils.convertLongToDate(e.getTimeStamp(), "ms") + ", " + e.getMessage();
pipeExceptionMessage2RegionIdsMap
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
index 97c6795daf4..718a9fe97e4 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
@@ -248,24 +248,21 @@ public class PipeHeartbeatParser {
// Update runtime exception
final PipeTaskMeta pipeTaskMetaFromCoordinator = runtimeMetaFromCoordinator.getValue();
+ final PipeRuntimeMeta pipeRuntimeMeta = pipeMetaFromCoordinator.getRuntimeMeta();
pipeTaskMetaFromCoordinator.clearExceptionMessages();
for (final PipeRuntimeException exception : runtimeMetaFromAgent.getExceptionMessages()) {
-
- // Do not judge the exception's clear time to avoid the restart process
- // being ended after the failure of some pipe
+ if (exception.getTimeStamp() <= pipeRuntimeMeta.getExceptionsClearTime()) {
+ needPushPipeMetaToDataNodes.set(true);
+ continue;
+ }
pipeTaskMetaFromCoordinator.trackExceptionMessage(exception);
if (exception instanceof PipeRuntimeCriticalException) {
final String pipeName = pipeMetaFromCoordinator.getStaticMeta().getPipeName();
- if (!pipeMetaFromCoordinator
- .getRuntimeMeta()
- .getStatus()
- .get()
- .equals(PipeStatus.STOPPED)) {
- PipeRuntimeMeta runtimeMeta = pipeMetaFromCoordinator.getRuntimeMeta();
- runtimeMeta.getStatus().set(PipeStatus.STOPPED);
- runtimeMeta.setIsStoppedByRuntimeException(true);
+ if (!pipeRuntimeMeta.getStatus().get().equals(PipeStatus.STOPPED)) {
+ pipeRuntimeMeta.getStatus().set(PipeStatus.STOPPED);
+ pipeRuntimeMeta.setIsStoppedByRuntimeException(true);
needWriteConsensusOnConfigNodes.set(true);
needPushPipeMetaToDataNodes.set(false);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
index 247f803152b..d231a613621 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
@@ -763,14 +763,26 @@ public class PipeTaskInfo implements SnapshotProcessor {
public void clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(final String pipeName) {
acquireWriteLock();
try {
- clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal(pipeName);
+ clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal(
+ pipeName, System.currentTimeMillis());
+ } finally {
+ releaseWriteLock();
+ }
+ }
+
+ public void clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(
+ final String pipeName, final long exceptionsClearTime) {
+ acquireWriteLock();
+ try {
+ clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal(
+ pipeName, exceptionsClearTime);
} finally {
releaseWriteLock();
}
}
private void clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal(
- final String pipeName) {
+ final String pipeName, final long exceptionsClearTime) {
if (!pipeMetaKeeper.containsPipeMeta(pipeName)) {
return;
}
@@ -780,7 +792,7 @@ public class PipeTaskInfo implements SnapshotProcessor {
// To avoid unnecessary retries, we set the isStoppedByRuntimeException flag to false
runtimeMeta.setIsStoppedByRuntimeException(false);
- runtimeMeta.setExceptionsClearTime(System.currentTimeMillis());
+ runtimeMeta.setExceptionsClearTime(exceptionsClearTime);
final Map<Integer, PipeRuntimeException> exceptionMap =
runtimeMeta.getNodeId2PipeRuntimeExceptionMap();
@@ -904,14 +916,17 @@ public class PipeTaskInfo implements SnapshotProcessor {
*/
private boolean autoRestartInternal() {
final AtomicBoolean needRestart = new AtomicBoolean(false);
+ final long exceptionsClearTime = System.currentTimeMillis();
final List<String> pipeToRestart = new LinkedList<>();
pipeMetaKeeper
.getPipeMetaList()
.forEach(
pipeMeta -> {
- if (pipeMeta.getRuntimeMeta().getIsStoppedByRuntimeException()) {
- pipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.RUNNING);
+ final PipeRuntimeMeta runtimeMeta = pipeMeta.getRuntimeMeta();
+ if (runtimeMeta.getIsStoppedByRuntimeException()) {
+ runtimeMeta.setExceptionsClearTime(exceptionsClearTime);
+ runtimeMeta.getStatus().set(PipeStatus.RUNNING);
needRestart.set(true);
pipeToRestart.add(pipeMeta.getStaticMeta().getPipeName());
@@ -945,9 +960,11 @@ public class PipeTaskInfo implements SnapshotProcessor {
.getPipeMetaList()
.forEach(
pipeMeta -> {
- if (pipeMeta.getRuntimeMeta().getStatus().get().equals(PipeStatus.RUNNING)) {
- clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(
- pipeMeta.getStaticMeta().getPipeName());
+ final PipeRuntimeMeta runtimeMeta = pipeMeta.getRuntimeMeta();
+ if (runtimeMeta.getStatus().get().equals(PipeStatus.RUNNING)
+ && runtimeMeta.getIsStoppedByRuntimeException()) {
+ clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal(
+ pipeMeta.getStaticMeta().getPipeName(), runtimeMeta.getExceptionsClearTime());
}
});
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java
index 7c9bff96f8f..42ac65a74c3 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java
@@ -20,7 +20,9 @@
package org.apache.iotdb.confignode.procedure.impl.pipe.task;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.runtime.PipeHandleMetaChangePlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2;
import org.apache.iotdb.confignode.i18n.ConfigNodeMessages;
import org.apache.iotdb.confignode.i18n.ProcedureMessages;
@@ -39,6 +41,8 @@ import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
public class StartPipeProcedureV2 extends AbstractOperatePipeProcedureV2 {
@@ -102,6 +106,9 @@ public class StartPipeProcedureV2 extends AbstractOperatePipeProcedureV2 {
public void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) throws IOException {
LOGGER.info(ProcedureMessages.STARTPIPEPROCEDUREV2_EXECUTEFROMOPERATEONDATANODES, pipeName);
+ final long exceptionsClearTime = System.currentTimeMillis();
+ final boolean isStoppedByRuntimeException =
+ pipeTaskInfo.get().isStoppedByRuntimeException(pipeName);
final String exceptionMessage =
parsePushPipeMetaExceptionForPipe(pipeName, pushSinglePipeMetaToDataNodes(pipeName, env));
if (!exceptionMessage.isEmpty()) {
@@ -114,7 +121,35 @@ public class StartPipeProcedureV2 extends AbstractOperatePipeProcedureV2 {
// Clear exceptions and set isStoppedByRuntimeException to false if the pipe is
// started successfully on all data nodes
- pipeTaskInfo.get().clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(pipeName);
+ pipeTaskInfo
+ .get()
+ .clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(pipeName, exceptionsClearTime);
+
+ if (isStoppedByRuntimeException) {
+ writePipeMetaChangesToConfigNodeConsensus(env);
+ }
+ }
+
+ private void writePipeMetaChangesToConfigNodeConsensus(final ConfigNodeProcedureEnv env) {
+ final List<PipeMeta> pipeMetaList = new ArrayList<>();
+ for (final PipeMeta pipeMeta : pipeTaskInfo.get().getPipeMetaList()) {
+ pipeMetaList.add(pipeMeta);
+ }
+
+ TSStatus response;
+ try {
+ response =
+ env.getConfigManager()
+ .getConsensusManager()
+ .write(new PipeHandleMetaChangePlan(pipeMetaList));
+ } catch (ConsensusException e) {
+ LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e);
+ response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
+ response.setMessage(e.getMessage());
+ }
+ if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new PipeException(response.getMessage());
+ }
}
@Override
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java
index d5a46d42c84..7383bf7937a 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java
@@ -20,6 +20,14 @@
package org.apache.iotdb.confignode.manager.pipe.coordinator.runtime.heartbeat;
import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
+import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.confignode.manager.ProcedureManager;
import org.apache.iotdb.confignode.manager.node.NodeManager;
@@ -29,13 +37,18 @@ import org.apache.iotdb.confignode.manager.pipe.coordinator.task.PipeTaskCoordin
import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import java.lang.reflect.Field;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -49,6 +62,8 @@ import static org.mockito.Mockito.when;
public class PipeHeartbeatParserTest {
+ private static final int DATA_NODE_ID = 1;
+
private boolean originalSeparatedPipeHeartbeatEnabled;
@Before
@@ -117,7 +132,88 @@ public class PipeHeartbeatParserTest {
verify(context.procedureManager, times(2)).pipeHandleMetaChange(true, false);
}
+ @Test
+ public void testParseHeartbeatIgnoresExceptionsBeforeClearTime() throws Exception {
+ CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false);
+
+ final String pipeName = "staleExceptionPipe";
+ final PipeTaskInfo pipeTaskInfo = new PipeTaskInfo();
+ createPipe(pipeTaskInfo, pipeName, PipeStatus.RUNNING);
+
+ final PipeMeta pipeMeta = pipeTaskInfo.getPipeMetaByPipeName(pipeName);
+ final PipeRuntimeMeta runtimeMeta = pipeMeta.getRuntimeMeta();
+ final PipeTaskMeta coordinatorTaskMeta =
+ runtimeMeta.getConsensusGroupId2TaskMetaMap().get(DATA_NODE_ID);
+ coordinatorTaskMeta.trackExceptionMessage(
+ new PipeRuntimeCriticalException("stale failure", 100L));
+
+ pipeTaskInfo.clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(pipeName, 200L);
+
+ final PipeTaskMeta agentTaskMeta =
+ new PipeTaskMeta(MinimumProgressIndex.INSTANCE, DATA_NODE_ID);
+ agentTaskMeta.trackExceptionMessage(new PipeRuntimeCriticalException("stale failure", 100L));
+ final ConcurrentMap<Integer, PipeTaskMeta> agentPipeTasks = new ConcurrentHashMap<>();
+ agentPipeTasks.put(DATA_NODE_ID, agentTaskMeta);
+ final PipeHeartbeat heartbeat =
+ new PipeHeartbeat(
+ Collections.singletonList(
+ new PipeMeta(pipeMeta.getStaticMeta(), new PipeRuntimeMeta(agentPipeTasks))
+ .serialize()),
+ Collections.singletonList(false),
+ Collections.singletonList(0L),
+ Collections.singletonList(0D));
+
+ final ParserTestContext context = createParserTestContext(1, pipeTaskInfo);
+ context.parser.parseHeartbeat(DATA_NODE_ID, heartbeat);
+
+ Assert.assertFalse(coordinatorTaskMeta.hasExceptionMessages());
+ Assert.assertEquals(PipeStatus.RUNNING, runtimeMeta.getStatus().get());
+ verify(context.procedureManager, times(1)).pipeHandleMetaChange(false, true);
+ }
+
+ @Test
+ public void testParseHeartbeatTracksExceptionsAfterClearTime() throws Exception {
+ CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false);
+
+ final String pipeName = "freshExceptionPipe";
+ final PipeTaskInfo pipeTaskInfo = new PipeTaskInfo();
+ createPipe(pipeTaskInfo, pipeName, PipeStatus.RUNNING);
+
+ final PipeMeta pipeMeta = pipeTaskInfo.getPipeMetaByPipeName(pipeName);
+ final PipeRuntimeMeta runtimeMeta = pipeMeta.getRuntimeMeta();
+ final PipeTaskMeta coordinatorTaskMeta =
+ runtimeMeta.getConsensusGroupId2TaskMetaMap().get(DATA_NODE_ID);
+ pipeTaskInfo.clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(pipeName, 200L);
+
+ final PipeTaskMeta agentTaskMeta =
+ new PipeTaskMeta(MinimumProgressIndex.INSTANCE, DATA_NODE_ID);
+ agentTaskMeta.trackExceptionMessage(new PipeRuntimeCriticalException("fresh failure", 300L));
+ final ConcurrentMap<Integer, PipeTaskMeta> agentPipeTasks = new ConcurrentHashMap<>();
+ agentPipeTasks.put(DATA_NODE_ID, agentTaskMeta);
+ final PipeHeartbeat heartbeat =
+ new PipeHeartbeat(
+ Collections.singletonList(
+ new PipeMeta(pipeMeta.getStaticMeta(), new PipeRuntimeMeta(agentPipeTasks))
+ .serialize()),
+ Collections.singletonList(false),
+ Collections.singletonList(0L),
+ Collections.singletonList(0D));
+
+ final ParserTestContext context = createParserTestContext(1, pipeTaskInfo);
+ context.parser.parseHeartbeat(DATA_NODE_ID, heartbeat);
+
+ Assert.assertTrue(coordinatorTaskMeta.hasExceptionMessages());
+ Assert.assertEquals(PipeStatus.STOPPED, runtimeMeta.getStatus().get());
+ Assert.assertTrue(runtimeMeta.getIsStoppedByRuntimeException());
+ verify(context.procedureManager, times(1)).pipeHandleMetaChange(true, false);
+ }
+
private ParserTestContext createParserTestContext(final int registeredDataNodeCount) {
+ return createParserTestContext(registeredDataNodeCount, new PipeTaskInfo());
+ }
+
+ private ParserTestContext createParserTestContext(
+ final int registeredDataNodeCount, final PipeTaskInfo pipeTaskInfo) {
final ConfigManager configManager = Mockito.mock(ConfigManager.class);
final NodeManager nodeManager = Mockito.mock(NodeManager.class);
final ProcedureManager procedureManager = Mockito.mock(ProcedureManager.class);
@@ -134,7 +230,7 @@ public class PipeHeartbeatParserTest {
when(pipeManager.getPipeRuntimeCoordinator()).thenReturn(pipeRuntimeCoordinator);
when(pipeManager.getPipeTaskCoordinator()).thenReturn(pipeTaskCoordinator);
when(pipeRuntimeCoordinator.getProcedureSubmitter()).thenReturn(procedureSubmitter);
- when(pipeTaskCoordinator.tryLock()).thenReturn(new AtomicReference<>(new PipeTaskInfo()));
+ when(pipeTaskCoordinator.tryLock()).thenReturn(new AtomicReference<>(pipeTaskInfo));
when(procedureManager.pipeHandleMetaChange(anyBoolean(), anyBoolean())).thenReturn(true);
Mockito.doAnswer(
invocation -> {
@@ -147,6 +243,37 @@ public class PipeHeartbeatParserTest {
return new ParserTestContext(new PipeHeartbeatParser(configManager), procedureManager);
}
+ private void createPipe(
+ final PipeTaskInfo pipeTaskInfo, final String pipeName, final PipeStatus initialStatus) {
+ final Map<String, String> extractorAttributes = new HashMap<>();
+ extractorAttributes.put("extractor", "iotdb-source");
+ final Map<String, String> processorAttributes = new HashMap<>();
+ processorAttributes.put("processor", "do-nothing-processor");
+ final Map<String, String> connectorAttributes = new HashMap<>();
+ connectorAttributes.put("connector", "iotdb-thrift-sink");
+
+ final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, DATA_NODE_ID);
+ final ConcurrentMap<Integer, PipeTaskMeta> pipeTasks = new ConcurrentHashMap<>();
+ pipeTasks.put(DATA_NODE_ID, pipeTaskMeta);
+ final PipeStaticMeta pipeStaticMeta =
+ new PipeStaticMeta(
+ pipeName,
+ System.currentTimeMillis(),
+ extractorAttributes,
+ processorAttributes,
+ connectorAttributes);
+ final PipeRuntimeMeta pipeRuntimeMeta = new PipeRuntimeMeta(pipeTasks);
+ pipeTaskInfo.createPipe(new CreatePipePlanV2(pipeStaticMeta, pipeRuntimeMeta));
+
+ if (PipeStatus.RUNNING.equals(initialStatus)) {
+ pipeTaskInfo
+ .getPipeMetaByPipeName(pipeName)
+ .getRuntimeMeta()
+ .getStatus()
+ .set(PipeStatus.RUNNING);
+ }
+ }
+
private void setMetaChangeFlags(
final PipeHeartbeatParser parser,
final boolean needWriteConsensusOnConfigNodes,
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java
index 7b78f59253d..39f0095d00e 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java
@@ -231,6 +231,34 @@ public class PipeTaskInfoAutoRestartTest {
rootPassword, sourceAttributes.get(PipeSourceConstant.SOURCE_IOTDB_PASSWORD_KEY));
}
+ @Test
+ public void testHandleSuccessfulRestartClearsRuntimeExceptionMessages() {
+ final String pipeName = "restartPipe";
+ createPipe(pipeName, PipeStatus.RUNNING);
+
+ Assert.assertTrue(
+ pipeTaskInfo.recordDataNodePushPipeMetaExceptions(createErrorRespMap(pipeName)));
+
+ final PipeRuntimeMeta runtimeMeta =
+ pipeTaskInfo.getPipeMetaByPipeName(pipeName).getRuntimeMeta();
+ Assert.assertEquals(PipeStatus.STOPPED, runtimeMeta.getStatus().get());
+ Assert.assertTrue(runtimeMeta.getIsStoppedByRuntimeException());
+ Assert.assertFalse(runtimeMeta.getNodeId2PipeRuntimeExceptionMap().isEmpty());
+
+ Assert.assertTrue(pipeTaskInfo.autoRestart());
+ final long exceptionsClearTime = runtimeMeta.getExceptionsClearTime();
+ Assert.assertTrue(
+ runtimeMeta.getNodeId2PipeRuntimeExceptionMap().values().stream()
+ .allMatch(exception -> exception.getTimeStamp() <= exceptionsClearTime));
+
+ pipeTaskInfo.handleSuccessfulRestart();
+
+ Assert.assertEquals(PipeStatus.RUNNING, runtimeMeta.getStatus().get());
+ Assert.assertFalse(runtimeMeta.getIsStoppedByRuntimeException());
+ Assert.assertTrue(runtimeMeta.getNodeId2PipeRuntimeExceptionMap().isEmpty());
+ Assert.assertEquals(exceptionsClearTime, runtimeMeta.getExceptionsClearTime());
+ }
+
private Map<Integer, TPushPipeMetaResp> createErrorRespMap(final String pipeName) {
final TPushPipeMetaRespExceptionMessage exceptionMessage =
new TPushPipeMetaRespExceptionMessage(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
index c30a50ac495..3daeec4ebd8 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
@@ -305,6 +305,8 @@ public abstract class PipeTaskAgent {
}
}
+ syncRuntimeExceptionClearTime(runtimeMetaFromCoordinator, runtimeMetaInAgent);
+
// 2. Handle pipe runtime meta status changes
final PipeStatus statusFromCoordinator = runtimeMetaFromCoordinator.getStatus().get();
final PipeStatus statusInAgent = runtimeMetaInAgent.getStatus().get();
@@ -347,6 +349,12 @@ public abstract class PipeTaskAgent {
}
}
+ private void syncRuntimeExceptionClearTime(
+ final PipeRuntimeMeta runtimeMetaFromCoordinator, final PipeRuntimeMeta runtimeMetaInAgent) {
+ runtimeMetaInAgent.setExceptionsClearTime(runtimeMetaFromCoordinator.getExceptionsClearTime());
+ runtimeMetaInAgent.clearExceptionMessagesBefore(runtimeMetaInAgent.getExceptionsClearTime());
+ }
+
protected abstract void thawRate(final String pipeName, final long creationTime);
protected abstract void freezeRate(final String pipeName, final long creationTime);
@@ -548,6 +556,11 @@ public abstract class PipeTaskAgent {
pipeMetaKeeper.addPipeMeta(pipeMetaFromCoordinator);
+ pipeMetaFromCoordinator
+ .getRuntimeMeta()
+ .clearExceptionMessagesBefore(
+ pipeMetaFromCoordinator.getRuntimeMeta().getExceptionsClearTime());
+
// If the pipe status from coordinator is RUNNING, we will start the pipe later.
return needToStartPipe;
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeRuntimeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeRuntimeMeta.java
index 1f28a24dd60..2aa1b65638f 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeRuntimeMeta.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeRuntimeMeta.java
@@ -128,6 +128,15 @@ public class PipeRuntimeMeta {
}
}
+ public void clearExceptionMessagesBefore(final long exceptionsClearTime) {
+ nodeId2PipeRuntimeExceptionMap
+ .entrySet()
+ .removeIf(entry -> entry.getValue().getTimeStamp() <= exceptionsClearTime);
+ consensusGroupId2TaskMetaMap
+ .values()
+ .forEach(pipeTaskMeta -> pipeTaskMeta.clearExceptionMessagesBefore(exceptionsClearTime));
+ }
+
public boolean getIsStoppedByRuntimeException() {
return isStoppedByRuntimeException.get();
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java
index 9584ca8cbab..e9939d7b2c6 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java
@@ -134,6 +134,10 @@ public class PipeTaskMeta {
exceptionMessages.clear();
}
+ public synchronized void clearExceptionMessagesBefore(final long exceptionsClearTime) {
+ exceptionMessages.removeIf(exception -> exception.getTimeStamp() <= exceptionsClearTime);
+ }
+
public synchronized void serialize(final OutputStream outputStream) throws IOException {
progressIndex.get().serialize(outputStream);
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaDeSerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaDeSerTest.java
index 1e29c96e090..fa0ac2c1a37 100644
--- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaDeSerTest.java
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaDeSerTest.java
@@ -151,4 +151,30 @@ public class PipeMetaDeSerTest {
final PipeMeta pipeMeta1 = PipeMeta.deserialize4Coordinator(byteBuffer);
Assert.assertEquals(pipeMeta, pipeMeta1);
}
+
+ @Test
+ public void testClearExceptionMessagesBeforeClearTime() {
+ final PipeTaskMeta staleTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
+ staleTaskMeta.trackExceptionMessage(new PipeRuntimeCriticalException("stale", 100L));
+ final PipeTaskMeta freshTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
+ freshTaskMeta.trackExceptionMessage(new PipeRuntimeCriticalException("fresh", 300L));
+
+ final ConcurrentHashMap<Integer, PipeTaskMeta> taskMetaMap = new ConcurrentHashMap<>();
+ taskMetaMap.put(1, staleTaskMeta);
+ taskMetaMap.put(2, freshTaskMeta);
+ final PipeRuntimeMeta runtimeMeta = new PipeRuntimeMeta(taskMetaMap);
+ runtimeMeta
+ .getNodeId2PipeRuntimeExceptionMap()
+ .put(1, new PipeRuntimeCriticalException("stale node", 100L));
+ runtimeMeta
+ .getNodeId2PipeRuntimeExceptionMap()
+ .put(2, new PipeRuntimeCriticalException("fresh node", 300L));
+
+ runtimeMeta.clearExceptionMessagesBefore(200L);
+
+ Assert.assertFalse(staleTaskMeta.hasExceptionMessages());
+ Assert.assertTrue(freshTaskMeta.hasExceptionMessages());
+ Assert.assertFalse(runtimeMeta.getNodeId2PipeRuntimeExceptionMap().containsKey(1));
+ Assert.assertTrue(runtimeMeta.getNodeId2PipeRuntimeExceptionMap().containsKey(2));
+ }
}