You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dubbo.apache.org by ox...@apache.org on 2025/10/31 03:50:10 UTC
(dubbo) branch 3.3-dev updated: feat: support reliable triple stream (#15712)
This is an automated email from the ASF dual-hosted git repository.
oxsean pushed a commit to branch 3.3-dev
in repository https://gitbox.apache.org/repos/asf/dubbo.git
The following commit(s) were added to refs/heads/3.3-dev by this push:
new 53eacd52a2 feat: support reliable triple stream (#15712)
53eacd52a2 is described below
commit 53eacd52a24e3833723fd3826c3d04057122c15b
Author: heliang <35...@qq.com>
AuthorDate: Fri Oct 31 11:50:04 2025 +0800
feat: support reliable triple stream (#15712)
* feat: support reliable triple stream
* code format
* code format
* fix log error
* fix log error
* fix log error
* fix ci
* fix ci
---
.../org/apache/dubbo/common/utils/NetUtils.java | 7 +-
.../apache/dubbo/common/utils/NetUtilsTest.java | 6 +
.../dubbo/rpc/protocol/tri/TripleInvoker.java | 11 +
.../rpc/protocol/tri/call/TripleClientCall.java | 142 +-
.../tri/call/TripleReconnectionManager.java | 174 ++
.../tri/command/CreateStreamQueueCommand.java | 28 +-
.../tri/h12/BiStreamServerCallListener.java | 302 ++
.../tri/h12/ServerStreamServerCallListener.java | 281 ++
.../http2/GenericHttp2ServerTransportListener.java | 150 +-
.../tri/h12/http2/Http2ServerStreamObserver.java | 50 +
.../protocol/tri/observer/CallStreamObserver.java | 9 +
.../tri/observer/ClientCallToObserverAdapter.java | 10 +
.../tri/server/FileServerSequenceStore.java | 388 +++
.../tri/server/InMemoryServerSequenceStore.java | 209 ++
.../tri/server/ServerSequenceException.java | 36 +
.../protocol/tri/server/ServerSequenceStore.java | 154 +
.../tri/store/FilePendingMessageStore.java | 663 ++++
.../tri/store/InMemoryPendingMessageStore.java | 188 ++
.../rpc/protocol/tri/store/PendingMessage.java | 140 +
.../protocol/tri/store/PendingMessageStore.java | 103 +
.../rpc/protocol/tri/store/SessionMetadata.java | 149 +
.../rpc/protocol/tri/store/StoreException.java | 35 +
.../tri/stream/AbstractTripleClientStream.java | 3297 +++++++++++++++++++-
.../protocol/tri/stream/ReconnectionManager.java | 59 +
.../rpc/protocol/tri/stream/ReliabilityConfig.java | 238 ++
.../protocol/tri/stream/ReliabilityContext.java | 141 +
...bbo.rpc.protocol.tri.server.ServerSequenceStore | 2 +
...ubbo.rpc.protocol.tri.store.PendingMessageStore | 2 +
28 files changed, 6897 insertions(+), 77 deletions(-)
diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/NetUtils.java b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/NetUtils.java
index 766a12427b..ea7958b83c 100644
--- a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/NetUtils.java
+++ b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/NetUtils.java
@@ -647,7 +647,12 @@ public final class NetUtils {
}
public static String toAddressString(InetSocketAddress address) {
- return address.getAddress().getHostAddress() + ":" + address.getPort();
+ if (address == null) {
+ return null;
+ }
+ InetAddress inetAddress = address.getAddress();
+ String host = inetAddress != null ? inetAddress.getHostAddress() : address.getHostString();
+ return host + ":" + address.getPort();
}
public static InetSocketAddress toAddress(String address) {
diff --git a/dubbo-common/src/test/java/org/apache/dubbo/common/utils/NetUtilsTest.java b/dubbo-common/src/test/java/org/apache/dubbo/common/utils/NetUtilsTest.java
index d5818d90a1..c992c6f830 100644
--- a/dubbo-common/src/test/java/org/apache/dubbo/common/utils/NetUtilsTest.java
+++ b/dubbo-common/src/test/java/org/apache/dubbo/common/utils/NetUtilsTest.java
@@ -177,6 +177,12 @@ class NetUtilsTest {
assertThat(NetUtils.toAddressString(socketAddress), equalTo("dubbo:1234"));
}
+ @Test
+ void testToAddressStringWhenUnresolved() {
+ InetSocketAddress socketAddress = InetSocketAddress.createUnresolved("dubbo-unresolved", 4321);
+ assertThat(NetUtils.toAddressString(socketAddress), equalTo("dubbo-unresolved:4321"));
+ }
+
@Test
void testToAddress() {
InetSocketAddress address = NetUtils.toAddress("localhost:1234");
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/TripleInvoker.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/TripleInvoker.java
index 1a6bac6073..21328e7828 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/TripleInvoker.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/TripleInvoker.java
@@ -61,6 +61,7 @@ import org.apache.dubbo.rpc.service.ServiceDescriptorInternalCache;
import org.apache.dubbo.rpc.support.RpcUtils;
import java.util.Arrays;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -337,6 +338,16 @@ public class TripleInvoker<T> extends AbstractInvoker<T> {
application = (String) invocation.getObjectAttachmentWithoutConvert(CommonConstants.REMOTE_APPLICATION_KEY);
}
meta.application = application;
+
+ // Add reliability negotiation for streaming calls (disabled by default for compatibility)
+ boolean reliabilityEnabled = getUrl().getParameter("stream.reliability.enabled", false);
+ if (reliabilityEnabled && methodDescriptor.getRpcType() != UNARY) {
+ Map<String, Object> attachments = invocation.getObjectAttachments();
+ attachments.put("tri-reliable-version", "1.0");
+ attachments.put("tri-session-id", java.util.UUID.randomUUID().toString());
+ attachments.put("tri-stream-cap", "ack,heartbeat,retry");
+ }
+
meta.attachments = invocation.getObjectAttachments();
return meta;
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/call/TripleClientCall.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/call/TripleClientCall.java
index 9965191497..ffb4375540 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/call/TripleClientCall.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/call/TripleClientCall.java
@@ -41,7 +41,8 @@ import static org.apache.dubbo.common.constants.LoggerCodeConstants.PROTOCOL_FAI
import static org.apache.dubbo.common.constants.LoggerCodeConstants.PROTOCOL_FAILED_SERIALIZE_TRIPLE;
import static org.apache.dubbo.common.constants.LoggerCodeConstants.PROTOCOL_STREAM_LISTENER;
-public class TripleClientCall implements ClientCall, ClientStream.Listener {
+public class TripleClientCall
+ implements ClientCall, ClientStream.Listener, org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext {
private static final ErrorTypeAwareLogger LOGGER = LoggerFactory.getErrorTypeAwareLogger(TripleClientCall.class);
private final AbstractConnectionClient connectionClient;
private final Executor executor;
@@ -253,6 +254,19 @@ public class TripleClientCall implements ClientCall, ClientStream.Listener {
this.requestMetadata = metadata;
this.listener = responseListener;
this.stream = stream;
+
+ // Initialize reliability if the stream supports it
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.AbstractTripleClientStream) {
+ org.apache.dubbo.rpc.protocol.tri.stream.AbstractTripleClientStream reliableStream =
+ (org.apache.dubbo.rpc.protocol.tri.stream.AbstractTripleClientStream) stream;
+ reliableStream.initializeReliability(metadata, connectionClient.getUrl());
+
+ // Set reconnection manager for real reconnection capability
+ // Use the existing transport listener to maintain consistency across reconnection
+ reliableStream.setReconnectionManager(new TripleReconnectionManager(
+ connectionClient, this, reliableStream.getTransportListener()));
+ }
+
return new ClientCallToObserverAdapter<>(this);
}
}
@@ -268,4 +282,130 @@ public class TripleClientCall implements ClientCall, ClientStream.Listener {
public void setAutoRequest(boolean autoRequest) {
this.autoRequest = autoRequest;
}
+
+ // ========== ReliabilityContext Implementation ==========
+
+ @Override
+ public String getState() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getState();
+ }
+ return "DISABLED";
+ }
+
+ @Override
+ public long getLastAckedSeq() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getLastAckedSeq();
+ }
+ return -1;
+ }
+
+ @Override
+ public int getPendingCount() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getPendingCount();
+ }
+ return 0;
+ }
+
+ @Override
+ public int getInFlightCount() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getInFlightCount();
+ }
+ return 0;
+ }
+
+ @Override
+ public String getSessionId() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getSessionId();
+ }
+ return null;
+ }
+
+ @Override
+ public long getTotalSentCount() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getTotalSentCount();
+ }
+ return 0;
+ }
+
+ @Override
+ public long getTotalRetryCount() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getTotalRetryCount();
+ }
+ return 0;
+ }
+
+ @Override
+ public boolean isConnectionActive() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).isConnectionActive();
+ }
+ return false;
+ }
+
+ @Override
+ public java.util.Map<String, Object> getStatistics() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getStatistics();
+ }
+ return new java.util.HashMap<>();
+ }
+
+ @Override
+ public void onStateChange(java.util.function.Consumer<String> callback) {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).onStateChange(callback);
+ }
+ }
+
+ @Override
+ public void onRecovery(java.util.function.Consumer<String> callback) {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).onRecovery(callback);
+ }
+ }
+
+ @Override
+ public void onRetry(java.util.function.Consumer<Long> callback) {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).onRetry(callback);
+ }
+ }
+
+ @Override
+ public boolean retryMessage(long sequence) {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).retryMessage(sequence);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean triggerRecovery() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).triggerRecovery();
+ }
+ return false;
+ }
+
+ @Override
+ public void setInFlightLimit(int limit) {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).setInFlightLimit(limit);
+ }
+ }
+
+ @Override
+ public org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityConfig getRetryConfig() {
+ if (stream instanceof org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) {
+ return ((org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext) stream).getRetryConfig();
+ }
+ return null;
+ }
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/call/TripleReconnectionManager.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/call/TripleReconnectionManager.java
new file mode 100644
index 0000000000..9800322dd1
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/call/TripleReconnectionManager.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.call;
+
+import org.apache.dubbo.common.constants.LoggerCodeConstants;
+import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.remoting.api.connection.AbstractConnectionClient;
+import org.apache.dubbo.rpc.protocol.tri.command.CreateStreamQueueCommand;
+import org.apache.dubbo.rpc.protocol.tri.stream.ReconnectionManager;
+import org.apache.dubbo.rpc.protocol.tri.stream.TripleStreamChannelFuture;
+import org.apache.dubbo.rpc.protocol.tri.transport.H2TransportListener;
+import org.apache.dubbo.rpc.protocol.tri.transport.TripleCommandOutBoundHandler;
+import org.apache.dubbo.rpc.protocol.tri.transport.TripleHttp2ClientResponseHandler;
+
+import java.util.concurrent.CompletableFuture;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.codec.http2.Http2StreamChannelBootstrap;
+
+/**
+ * Implementation of ReconnectionManager that uses AbstractConnectionClient for reconnection.
+ * This provides real reconnection capability by leveraging existing connection management.
+ */
+public class TripleReconnectionManager implements ReconnectionManager {
+
+ private static final ErrorTypeAwareLogger LOGGER =
+ LoggerFactory.getErrorTypeAwareLogger(TripleReconnectionManager.class);
+
+ private final AbstractConnectionClient connectionClient;
+ private final TripleClientCall clientCall;
+ private final H2TransportListener transportListener;
+
+ public TripleReconnectionManager(
+ AbstractConnectionClient connectionClient,
+ TripleClientCall clientCall,
+ H2TransportListener transportListener) {
+ this.connectionClient = connectionClient;
+ this.clientCall = clientCall;
+ this.transportListener = transportListener;
+ }
+
+ @Override
+ public CompletableFuture<Boolean> attemptReconnection() {
+ return CompletableFuture.supplyAsync(() -> {
+ try {
+ LOGGER.info("Attempting reconnection using connection client: {}", connectionClient);
+
+ // Force get a new channel, which will trigger reconnection if needed
+ Object channel = connectionClient.getChannel(true);
+
+ boolean success = (channel != null) && isConnectionActive();
+
+ if (success) {
+ LOGGER.info("Reconnection successful for connection client: {}", connectionClient);
+ } else {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Reconnection failed for connection client: " + connectionClient);
+ }
+
+ return success;
+ } catch (Exception e) {
+ LOGGER.error(LoggerCodeConstants.INTERNAL_ERROR, "", "", "Error during reconnection attempt", e);
+ return false;
+ }
+ });
+ }
+
+ @Override
+ public boolean isConnectionActive() {
+ try {
+ return connectionClient.isConnected();
+ } catch (Exception e) {
+ LOGGER.debug("Error checking connection status", e);
+ return false;
+ }
+ }
+
+ @Override
+ public Object getNewStreamChannel() {
+ try {
+ // Get the active channel, this should be the new channel after reconnection
+ return connectionClient.getChannel(true);
+ } catch (Exception e) {
+ LOGGER.error(LoggerCodeConstants.INTERNAL_ERROR, "", "", "Error getting new stream channel", e);
+ return null;
+ }
+ }
+
+ @Override
+ public TripleStreamChannelFuture createStreamFuture() {
+ try {
+ LOGGER.info("Creating new TripleStreamChannelFuture for reconnection");
+
+ // Force get a new channel, which will trigger reconnection if needed
+ Object channel = connectionClient.getChannel(true);
+
+ if (channel != null && isConnectionActive()) {
+ // Create a new Http2StreamChannel using the same mechanism as initial connection
+ Channel parentChannel = (Channel) channel;
+
+ // Create Http2StreamChannelBootstrap to open a new stream channel
+ Http2StreamChannelBootstrap bootstrap = new Http2StreamChannelBootstrap(parentChannel);
+
+ // Set up the handler to configure the pipeline when the stream channel is created
+ bootstrap.handler(new ChannelInboundHandlerAdapter() {
+ @Override
+ public void handlerAdded(ChannelHandlerContext ctx) {
+ ctx.channel()
+ .pipeline()
+ .addLast(new TripleCommandOutBoundHandler())
+ .addLast(new TripleHttp2ClientResponseHandler(transportListener));
+ }
+ });
+
+ // Create the future that will be completed when the stream channel is ready
+ TripleStreamChannelFuture streamChannelFuture = new TripleStreamChannelFuture(parentChannel);
+
+ // Use CreateStreamQueueCommand to create the actual Http2StreamChannel
+ CreateStreamQueueCommand createCommand =
+ CreateStreamQueueCommand.create(bootstrap, streamChannelFuture);
+
+ // IMPROVED: Execute the creation command and add better error handling
+ if (parentChannel.eventLoop().inEventLoop()) {
+ // If we're already in the event loop, execute directly
+ createCommand.run(parentChannel);
+ } else {
+ // Execute in the channel's event loop to ensure thread safety
+ parentChannel.eventLoop().execute(() -> createCommand.run(parentChannel));
+ }
+
+ // Return the future - caller is now responsible for waiting for completion
+ LOGGER.info("Started async creation of new Http2StreamChannel for reconnection, "
+ + "future will be completed when stream is ready");
+ return streamChannelFuture;
+
+ } else {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to create TripleStreamChannelFuture - no active channel available");
+ return null;
+ }
+ } catch (Exception e) {
+ LOGGER.error(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Error creating new TripleStreamChannelFuture for reconnection",
+ e);
+ return null;
+ }
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/command/CreateStreamQueueCommand.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/command/CreateStreamQueueCommand.java
index 7c53c58998..a7b616c3e1 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/command/CreateStreamQueueCommand.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/command/CreateStreamQueueCommand.java
@@ -51,12 +51,26 @@ public class CreateStreamQueueCommand extends QueuedCommand {
@Override
public void run(Channel channel) {
- // work in I/O thread
- Future<Http2StreamChannel> future = bootstrap.open();
- if (future.isSuccess()) {
- streamChannelFuture.complete(future.getNow());
- } else {
- streamChannelFuture.completeExceptionally(future.cause());
- }
+ // Execute in the channel's event loop to ensure thread safety
+ channel.eventLoop().execute(() -> {
+ // work in I/O thread
+ Future<Http2StreamChannel> future = bootstrap.open();
+
+ // Use async listener to handle the result properly
+ future.addListener((io.netty.util.concurrent.Future<Http2StreamChannel> f) -> {
+ if (f.isSuccess()) {
+ Http2StreamChannel streamChannel = f.getNow();
+ streamChannelFuture.complete(streamChannel);
+ } else {
+ Throwable cause = f.cause();
+ if (cause != null) {
+ streamChannelFuture.completeExceptionally(cause);
+ } else {
+ streamChannelFuture.completeExceptionally(
+ new RuntimeException("Failed to create Http2StreamChannel: unknown error"));
+ }
+ }
+ });
+ });
}
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/BiStreamServerCallListener.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/BiStreamServerCallListener.java
index 94baf72350..ee3be9b230 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/BiStreamServerCallListener.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/BiStreamServerCallListener.java
@@ -16,23 +16,113 @@
*/
package org.apache.dubbo.rpc.protocol.tri.h12;
+import org.apache.dubbo.common.constants.LoggerCodeConstants;
+import org.apache.dubbo.common.extension.ExtensionLoader;
+import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
+import org.apache.dubbo.common.logger.LoggerFactory;
import org.apache.dubbo.common.stream.StreamObserver;
import org.apache.dubbo.remoting.http12.FlowControlStreamObserver;
import org.apache.dubbo.remoting.http12.exception.HttpStatusException;
import org.apache.dubbo.rpc.Invoker;
import org.apache.dubbo.rpc.RpcInvocation;
+import org.apache.dubbo.rpc.protocol.tri.server.ServerSequenceException;
+import org.apache.dubbo.rpc.protocol.tri.server.ServerSequenceStore;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
public class BiStreamServerCallListener extends AbstractServerCallListener {
+ private static final ErrorTypeAwareLogger LOGGER =
+ LoggerFactory.getErrorTypeAwareLogger(BiStreamServerCallListener.class);
+
private StreamObserver<Object> requestObserver;
+ private final boolean reliabilityEnabled;
+ private final String sessionId;
+ private final AtomicLong lastAckedSeq;
+ private volatile Long currentSequence;
+
+ // 序列号存储接口(支持持久化)
+ private ServerSequenceStore sequenceStore;
+
+ // 幂等性支持:已处理序列号的滑动窗口(内存缓存)
+ private final ConcurrentHashMap<Long, Boolean> processedSequences;
+ private final AtomicLong minProcessedSeq; // 最小已处理序列号,用于窗口清理
+ private static final int MAX_WINDOW_SIZE = 10000; // 最大窗口大小
public BiStreamServerCallListener(
RpcInvocation invocation, Invoker<?> invoker, FlowControlStreamObserver<Object> responseObserver) {
super(invocation, invoker, responseObserver);
+ this.reliabilityEnabled = false;
+ this.sessionId = null;
+ this.lastAckedSeq = new AtomicLong(0);
+ this.processedSequences = new ConcurrentHashMap<>();
+ this.minProcessedSeq = new AtomicLong(0);
+ this.sequenceStore = null; // 不可靠模式下不使用序列号存储
+ invocation.setArguments(new Object[] {responseObserver});
+ invoke();
+ }
+
+ public BiStreamServerCallListener(
+ RpcInvocation invocation,
+ Invoker<?> invoker,
+ FlowControlStreamObserver<Object> responseObserver,
+ boolean reliabilityEnabled,
+ String sessionId) {
+ super(invocation, invoker, responseObserver);
+ this.reliabilityEnabled = reliabilityEnabled;
+ this.sessionId = sessionId;
+ this.lastAckedSeq = new AtomicLong(0);
+ this.processedSequences = new ConcurrentHashMap<>();
+ this.minProcessedSeq = new AtomicLong(0);
+
+ // 初始化序列号存储(仅在可靠模式下)
+ if (reliabilityEnabled) {
+ initializeSequenceStore(invocation);
+ } else {
+ this.sequenceStore = null;
+ }
+
invocation.setArguments(new Object[] {responseObserver});
invoke();
}
+ public void setCurrentSequence(Long sequence) {
+ this.currentSequence = sequence;
+ }
+
+ /**
+ * 初始化序列号存储
+ */
+ private void initializeSequenceStore(RpcInvocation invocation) {
+ try {
+ // 从URL参数获取存储类型配置,默认为内存存储
+ String storeType = invoker.getUrl().getParameter("tri.server.sequence.store.type", "memory");
+
+ ExtensionLoader<ServerSequenceStore> loader = ExtensionLoader.getExtensionLoader(ServerSequenceStore.class);
+ this.sequenceStore = loader.getExtension(storeType);
+
+ String serviceKey = invocation.getTargetServiceUniqueName() + "." + invocation.getMethodName();
+ sequenceStore.init(sessionId, serviceKey);
+
+ LOGGER.info(
+ "Initialized server sequence store of type: {} for session: {} from URL parameter",
+ storeType,
+ sessionId);
+ } catch (Exception e) {
+ LOGGER.error(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to initialize sequence store for session: " + sessionId,
+ e);
+ // 降级到内存窗口模式
+ this.sequenceStore = null;
+ }
+ }
+
@Override
@SuppressWarnings("unchecked")
public void onReturn(Object value) {
@@ -41,6 +131,20 @@ public class BiStreamServerCallListener extends AbstractServerCallListener {
@Override
public void onMessage(Object message) {
+ // Process reliability ACK first
+ if (reliabilityEnabled && currentSequence != null) {
+ // 检查幂等性:如果序列号已经处理过,直接ACK并跳过
+ if (isDuplicateMessage(currentSequence)) {
+ sendAck(currentSequence);
+ return;
+ }
+
+ // 记录已处理的序列号(同时更新内存和持久化存储)
+ markSequenceAsProcessed(currentSequence);
+ lastAckedSeq.set(currentSequence);
+ sendAck(currentSequence);
+ }
+
if (message instanceof Object[]) {
message = ((Object[]) message)[0];
}
@@ -50,6 +154,183 @@ public class BiStreamServerCallListener extends AbstractServerCallListener {
}
}
+ private void sendAck(long sequence) {
+ if (responseObserver instanceof AttachmentHolder) {
+ // Merge with existing attachments to avoid overwriting
+ Map<String, Object> existingAttachments = ((AttachmentHolder) responseObserver).getResponseAttachments();
+ Map<String, Object> ackAttachments =
+ existingAttachments != null ? new HashMap<>(existingAttachments) : new HashMap<>();
+
+ ackAttachments.put("tri-ack", String.valueOf(sequence));
+ ackAttachments.put("tri-last-acked", String.valueOf(lastAckedSeq.get()));
+
+ // 添加更丰富的状态信息
+ if (sequenceStore != null && sequenceStore.isHealthy()) {
+ try {
+ ServerSequenceStore.StoreStats stats = sequenceStore.getStats();
+ ackAttachments.put("tri-server-stats", stats.toString());
+
+ // 添加服务端处理窗口信息
+ ackAttachments.put("tri-server-window-info", getWindowInfo());
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to get server stats for session: " + sessionId,
+ e);
+ }
+ }
+
+ ((AttachmentHolder) responseObserver).setResponseAttachments(ackAttachments);
+
+ try {
+ // Use specialized method to send ACK headers immediately via HTTP/2 HEADERS frame
+ if (responseObserver instanceof org.apache.dubbo.rpc.protocol.tri.h12.http2.Http2ServerStreamObserver) {
+ ((org.apache.dubbo.rpc.protocol.tri.h12.http2.Http2ServerStreamObserver) responseObserver)
+ .sendAckHeaders(ackAttachments);
+ LOGGER.debug(
+ "Sent ACK headers for sequence: {}, tri-ack: {}, tri-last-acked: {}",
+ sequence,
+ sequence,
+ lastAckedSeq.get());
+ } else {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Cannot send ACK headers for sequence: " + sequence
+ + " - responseObserver is not Http2ServerStreamObserver: "
+ + responseObserver.getClass().getSimpleName());
+ }
+ } catch (Exception e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to send ACK headers for sequence: " + sequence,
+ e);
+ }
+ }
+ }
+
+ /**
+ * 检查是否为重复消息
+ */
+ private boolean isDuplicateMessage(Long sequence) {
+ if (sequence == null) {
+ return false;
+ }
+
+ // 首先检查持久化存储(如果有)
+ if (sequenceStore != null && sequenceStore.isHealthy()) {
+ try {
+ if (sequenceStore.isSequenceProcessed(sequence)) {
+ return true;
+ }
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to check sequence from persistent store: " + sequence,
+ e);
+ // 降级到内存检查
+ }
+ }
+
+ // 如果序列号小于最小已处理序列号,认为是重复的(已在窗口清理时被移除)
+ if (sequence <= minProcessedSeq.get()) {
+ return true;
+ }
+
+ // 检查是否在已处理序列号窗口中
+ return processedSequences.containsKey(sequence);
+ }
+
+ /**
+ * 标记序列号为已处理
+ */
+ private void markSequenceAsProcessed(Long sequence) {
+ if (sequence == null) {
+ return;
+ }
+
+ // 首先记录到持久化存储(如果有)
+ if (sequenceStore != null && sequenceStore.isHealthy()) {
+ try {
+ sequenceStore.recordSequence(sequence);
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to record sequence to persistent store: " + sequence,
+ e);
+ // 继续处理内存存储,不中断流程
+ }
+ }
+
+ // 同时记录到内存窗口(用于快速访问和窗口管理)
+ processedSequences.put(sequence, true);
+
+ // 清理滑动窗口,避免内存无限增长
+ cleanupSequenceWindow();
+ }
+
+ /**
+ * 清理序列号窗口,保持窗口大小在限制范围内
+ */
+ private void cleanupSequenceWindow() {
+ long currentSeq = currentSequence != null ? currentSequence : 0;
+ long minSeq = minProcessedSeq.get();
+
+ // 如果窗口大小超过限制,清理旧的序列号
+ if (processedSequences.size() > MAX_WINDOW_SIZE && currentSeq > minSeq + MAX_WINDOW_SIZE) {
+ long newMinSeq = currentSeq - MAX_WINDOW_SIZE;
+
+ // 同步清理持久化存储(使用新的cleanupAcknowledged方法)
+ if (sequenceStore != null && sequenceStore.isHealthy()) {
+ try {
+ sequenceStore.cleanupAcknowledged(newMinSeq);
+ LOGGER.debug("Synced persistent store cleanup up to: {} for session: {}", newMinSeq, sessionId);
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to cleanup persistent store up to: " + newMinSeq,
+ e);
+ // 继续清理内存存储,保持系统可用性
+ }
+ }
+
+ // 移除小于等于新最小序列号的记录
+ processedSequences.entrySet().removeIf(entry -> entry.getKey() <= newMinSeq);
+
+ // 更新最小序列号
+ minProcessedSeq.compareAndSet(minSeq, newMinSeq);
+
+ LOGGER.debug("Cleaned up sequence window: minSeq={}, windowSize={}", newMinSeq, processedSequences.size());
+ }
+ }
+
+ /**
+ * 获取已处理序列号数量(用于监控)
+ */
+ public int getProcessedSequenceCount() {
+ return processedSequences.size();
+ }
+
+ /**
+ * 获取当前处理窗口信息(用于监控和调试)
+ */
+ public String getWindowInfo() {
+ return String.format(
+ "minSeq=%d, currentSeq=%s, windowSize=%d",
+ minProcessedSeq.get(), currentSequence, processedSequences.size());
+ }
+
@Override
public void onCancel(long code) {
requestObserver.onError(new HttpStatusException((int) code));
@@ -57,6 +338,27 @@ public class BiStreamServerCallListener extends AbstractServerCallListener {
@Override
public void onComplete() {
+ // 清理序列号存储资源
+ cleanupSequenceStore();
requestObserver.onCompleted();
}
+
+ /**
+ * 清理序列号存储资源
+ */
+ private void cleanupSequenceStore() {
+ if (sequenceStore != null) {
+ try {
+ sequenceStore.close();
+ LOGGER.debug("Closed sequence store for session: {}", sessionId);
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Error closing sequence store for session: " + sessionId,
+ e);
+ }
+ }
+ }
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/ServerStreamServerCallListener.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/ServerStreamServerCallListener.java
index 8315ff7aba..00da411880 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/ServerStreamServerCallListener.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/ServerStreamServerCallListener.java
@@ -16,17 +16,105 @@
*/
package org.apache.dubbo.rpc.protocol.tri.h12;
+import org.apache.dubbo.common.constants.LoggerCodeConstants;
+import org.apache.dubbo.common.extension.ExtensionLoader;
+import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
+import org.apache.dubbo.common.logger.LoggerFactory;
import org.apache.dubbo.common.stream.StreamObserver;
import org.apache.dubbo.remoting.http12.HttpResult;
import org.apache.dubbo.remoting.http12.exception.HttpStatusException;
import org.apache.dubbo.rpc.Invoker;
import org.apache.dubbo.rpc.RpcInvocation;
+import org.apache.dubbo.rpc.protocol.tri.server.ServerSequenceException;
+import org.apache.dubbo.rpc.protocol.tri.server.ServerSequenceStore;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
public class ServerStreamServerCallListener extends AbstractServerCallListener {
+ private static final ErrorTypeAwareLogger LOGGER =
+ LoggerFactory.getErrorTypeAwareLogger(ServerStreamServerCallListener.class);
+
+ private final boolean reliabilityEnabled;
+ private final String sessionId;
+ private final AtomicLong lastAckedSeq;
+ private volatile Long currentSequence;
+
+ // 序列号存储接口(支持持久化)
+ private ServerSequenceStore sequenceStore;
+
+ // 幂等性支持:已处理序列号的滑动窗口(内存缓存)
+ private final ConcurrentHashMap<Long, Boolean> processedSequences;
+ private final AtomicLong minProcessedSeq; // 最小已处理序列号,用于窗口清理
+ private static final int MAX_WINDOW_SIZE = 10000; // 最大窗口大小
+
public ServerStreamServerCallListener(
RpcInvocation invocation, Invoker<?> invoker, StreamObserver<Object> responseObserver) {
super(invocation, invoker, responseObserver);
+ this.reliabilityEnabled = false;
+ this.sessionId = null;
+ this.lastAckedSeq = new AtomicLong(0);
+ this.processedSequences = new ConcurrentHashMap<>();
+ this.minProcessedSeq = new AtomicLong(0);
+ this.sequenceStore = null; // 不可靠模式下不使用序列号存储
+ }
+
+ public ServerStreamServerCallListener(
+ RpcInvocation invocation,
+ Invoker<?> invoker,
+ StreamObserver<Object> responseObserver,
+ boolean reliabilityEnabled,
+ String sessionId) {
+ super(invocation, invoker, responseObserver);
+ this.reliabilityEnabled = reliabilityEnabled;
+ this.sessionId = sessionId;
+ this.lastAckedSeq = new AtomicLong(0);
+ this.processedSequences = new ConcurrentHashMap<>();
+ this.minProcessedSeq = new AtomicLong(0);
+
+ // 初始化序列号存储(仅在可靠模式下)
+ if (reliabilityEnabled) {
+ initializeSequenceStore(invocation);
+ } else {
+ this.sequenceStore = null;
+ }
+ }
+
+ public void setCurrentSequence(Long sequence) {
+ this.currentSequence = sequence;
+ }
+
+ /**
+ * 初始化序列号存储
+ */
+ private void initializeSequenceStore(RpcInvocation invocation) {
+ try {
+ // 从URL参数获取存储类型配置,默认为内存存储
+ String storeType = invoker.getUrl().getParameter("tri.server.sequence.store.type", "memory");
+
+ ExtensionLoader<ServerSequenceStore> loader = ExtensionLoader.getExtensionLoader(ServerSequenceStore.class);
+ this.sequenceStore = loader.getExtension(storeType);
+
+ String serviceKey = invocation.getTargetServiceUniqueName() + "." + invocation.getMethodName();
+ sequenceStore.init(sessionId, serviceKey);
+
+ LOGGER.info(
+ "Initialized server sequence store of type: {} for session: {} from URL parameter",
+ storeType,
+ sessionId);
+ } catch (Exception e) {
+ LOGGER.error(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to initialize sequence store for session: " + sessionId,
+ e);
+ // 降级到内存窗口模式
+ this.sequenceStore = null;
+ }
}
@Override
@@ -39,6 +127,20 @@ public class ServerStreamServerCallListener extends AbstractServerCallListener {
@Override
public void onMessage(Object message) {
+ // Process reliability ACK first
+ if (reliabilityEnabled && currentSequence != null) {
+ // 检查幂等性:如果序列号已经处理过,直接ACK并跳过
+ if (isDuplicateMessage(currentSequence)) {
+ sendAck(currentSequence);
+ return;
+ }
+
+ // 记录已处理的序列号
+ markSequenceAsProcessed(currentSequence);
+ lastAckedSeq.set(currentSequence);
+ sendAck(currentSequence);
+ }
+
Class<?>[] params = invocation.getParameterTypes();
if (params.length == 1) {
if (params[0].isInstance(responseObserver)) {
@@ -52,6 +154,164 @@ public class ServerStreamServerCallListener extends AbstractServerCallListener {
invocation.setArguments(new Object[] {message, responseObserver});
}
+ private void sendAck(long sequence) {
+ if (responseObserver instanceof AttachmentHolder) {
+ // Merge with existing attachments to avoid overwriting
+ Map<String, Object> existingAttachments = ((AttachmentHolder) responseObserver).getResponseAttachments();
+ Map<String, Object> ackAttachments =
+ existingAttachments != null ? new HashMap<>(existingAttachments) : new HashMap<>();
+
+ ackAttachments.put("tri-ack", String.valueOf(sequence));
+ ackAttachments.put("tri-last-acked", String.valueOf(lastAckedSeq.get()));
+ ((AttachmentHolder) responseObserver).setResponseAttachments(ackAttachments);
+
+ try {
+ // Use specialized method to send ACK headers immediately via HTTP/2 HEADERS frame
+ if (responseObserver instanceof org.apache.dubbo.rpc.protocol.tri.h12.http2.Http2ServerStreamObserver) {
+ ((org.apache.dubbo.rpc.protocol.tri.h12.http2.Http2ServerStreamObserver) responseObserver)
+ .sendAckHeaders(ackAttachments);
+ LOGGER.debug(
+ "Sent ACK headers for sequence: {}, tri-ack: {}, tri-last-acked: {}",
+ sequence,
+ sequence,
+ lastAckedSeq.get());
+ } else {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Cannot send ACK headers for sequence: " + sequence
+ + " - responseObserver is not Http2ServerStreamObserver: "
+ + responseObserver.getClass().getSimpleName());
+ }
+ } catch (Exception e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to send ACK headers for sequence: " + sequence,
+ e);
+ }
+ }
+ }
+
+ /**
+ * 检查是否为重复消息
+ */
+ private boolean isDuplicateMessage(Long sequence) {
+ if (sequence == null) {
+ return false;
+ }
+
+ // 首先检查持久化存储(如果有)
+ if (sequenceStore != null && sequenceStore.isHealthy()) {
+ try {
+ if (sequenceStore.isSequenceProcessed(sequence)) {
+ return true;
+ }
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to check sequence from persistent store: " + sequence,
+ e);
+ // 降级到内存检查
+ }
+ }
+
+ // 如果序列号小于最小已处理序列号,认为是重复的(已在窗口清理时被移除)
+ if (sequence <= minProcessedSeq.get()) {
+ return true;
+ }
+
+ // 检查是否在已处理序列号窗口中
+ return processedSequences.containsKey(sequence);
+ }
+
+ /**
+ * 标记序列号为已处理
+ */
+ private void markSequenceAsProcessed(Long sequence) {
+ if (sequence == null) {
+ return;
+ }
+
+ // 首先记录到持久化存储(如果有)
+ if (sequenceStore != null && sequenceStore.isHealthy()) {
+ try {
+ sequenceStore.recordSequence(sequence);
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to record sequence to persistent store: " + sequence,
+ e);
+ // 继续处理内存存储,不中断流程
+ }
+ }
+
+ // 同时记录到内存窗口(用于快速访问和窗口管理)
+ processedSequences.put(sequence, true);
+
+ // 清理滑动窗口,避免内存无限增长
+ cleanupSequenceWindow();
+ }
+
+ /**
+ * 清理序列号窗口,保持窗口大小在限制范围内
+ */
+ private void cleanupSequenceWindow() {
+ long currentSeq = currentSequence != null ? currentSequence : 0;
+ long minSeq = minProcessedSeq.get();
+
+ // 如果窗口大小超过限制,清理旧的序列号
+ if (processedSequences.size() > MAX_WINDOW_SIZE && currentSeq > minSeq + MAX_WINDOW_SIZE) {
+ long newMinSeq = currentSeq - MAX_WINDOW_SIZE;
+
+ // 同步清理持久化存储(使用新的cleanupAcknowledged方法)
+ if (sequenceStore != null && sequenceStore.isHealthy()) {
+ try {
+ sequenceStore.cleanupAcknowledged(newMinSeq);
+ LOGGER.debug("Synced persistent store cleanup up to: {} for session: {}", newMinSeq, sessionId);
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to cleanup persistent store up to: " + newMinSeq,
+ e);
+ // 继续清理内存存储,保持系统可用性
+ }
+ }
+
+ // 移除小于等于新最小序列号的记录
+ processedSequences.entrySet().removeIf(entry -> entry.getKey() <= newMinSeq);
+
+ // 更新最小序列号
+ minProcessedSeq.compareAndSet(minSeq, newMinSeq);
+
+ LOGGER.debug("Cleaned up sequence window: minSeq={}, windowSize={}", newMinSeq, processedSequences.size());
+ }
+ }
+
+ /**
+ * 获取已处理序列号数量(用于监控)
+ */
+ public int getProcessedSequenceCount() {
+ return processedSequences.size();
+ }
+
+ /**
+ * 获取当前处理窗口信息(用于监控和调试)
+ */
+ public String getWindowInfo() {
+ return String.format(
+ "minSeq=%d, currentSeq=%s, windowSize=%d",
+ minProcessedSeq.get(), currentSequence, processedSequences.size());
+ }
+
@Override
public void onCancel(long code) {
responseObserver.onError(new HttpStatusException((int) code));
@@ -59,6 +319,27 @@ public class ServerStreamServerCallListener extends AbstractServerCallListener {
@Override
public void onComplete() {
+ // 清理序列号存储资源
+ cleanupSequenceStore();
invoke();
}
+
+ /**
+ * 清理序列号存储资源
+ */
+ private void cleanupSequenceStore() {
+ if (sequenceStore != null) {
+ try {
+ sequenceStore.close();
+ LOGGER.debug("Closed sequence store for session: {}", sessionId);
+ } catch (ServerSequenceException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Error closing sequence store for session: " + sessionId,
+ e);
+ }
+ }
+ }
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/http2/GenericHttp2ServerTransportListener.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/http2/GenericHttp2ServerTransportListener.java
index f4f0aeefb3..bd1e98b691 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/http2/GenericHttp2ServerTransportListener.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/http2/GenericHttp2ServerTransportListener.java
@@ -17,14 +17,17 @@
package org.apache.dubbo.rpc.protocol.tri.h12.http2;
import org.apache.dubbo.common.URL;
+import org.apache.dubbo.common.constants.LoggerCodeConstants;
import org.apache.dubbo.remoting.Constants;
import org.apache.dubbo.remoting.http12.HttpHeaderNames;
import org.apache.dubbo.remoting.http12.h2.CancelStreamException;
import org.apache.dubbo.remoting.http12.h2.H2StreamChannel;
import org.apache.dubbo.remoting.http12.h2.Http2Header;
import org.apache.dubbo.remoting.http12.h2.Http2InputMessage;
+import org.apache.dubbo.remoting.http12.h2.Http2MetadataFrame;
import org.apache.dubbo.remoting.http12.h2.Http2ServerChannelObserver;
import org.apache.dubbo.remoting.http12.h2.Http2TransportListener;
+import org.apache.dubbo.remoting.http12.message.DefaultHttpHeaders;
import org.apache.dubbo.remoting.http12.message.DefaultListeningDecoder;
import org.apache.dubbo.remoting.http12.message.DefaultStreamingDecoder;
import org.apache.dubbo.remoting.http12.message.ListeningDecoder;
@@ -39,6 +42,7 @@ import org.apache.dubbo.rpc.model.MethodDescriptor;
import org.apache.dubbo.rpc.protocol.tri.Http3Exchanger;
import org.apache.dubbo.rpc.protocol.tri.RpcInvocationBuildContext;
import org.apache.dubbo.rpc.protocol.tri.h12.AbstractServerTransportListener;
+import org.apache.dubbo.rpc.protocol.tri.h12.AttachmentHolder;
import org.apache.dubbo.rpc.protocol.tri.h12.BiStreamServerCallListener;
import org.apache.dubbo.rpc.protocol.tri.h12.HttpMessageListener;
import org.apache.dubbo.rpc.protocol.tri.h12.ServerCallListener;
@@ -50,6 +54,10 @@ import java.io.InputStream;
public class GenericHttp2ServerTransportListener extends AbstractServerTransportListener<Http2Header, Http2InputMessage>
implements Http2TransportListener {
+ private static final org.apache.dubbo.common.logger.ErrorTypeAwareLogger LOGGER =
+ org.apache.dubbo.common.logger.LoggerFactory.getErrorTypeAwareLogger(
+ GenericHttp2ServerTransportListener.class);
+
private final H2StreamChannel h2StreamChannel;
private final StreamingDecoder streamingDecoder;
private Http2ServerChannelObserver responseObserver;
@@ -96,24 +104,38 @@ public class GenericHttp2ServerTransportListener extends AbstractServerTransport
serverCallListener = startListener(rpcInvocation, context.getMethodDescriptor(), context.getInvoker());
DefaultListeningDecoder listeningDecoder = new DefaultListeningDecoder(
context.getHttpMessageDecoder(), context.getMethodMetadata().getActualRequestTypes());
- listeningDecoder.setListener(new Http2StreamingDecodeListener(serverCallListener));
+ listeningDecoder.setListener(new Http2StreamingDecodeListener(serverCallListener, this));
streamingDecoder.setFragmentListener(new StreamingDecoder.DefaultFragmentListener(listeningDecoder));
return new StreamingHttpMessageListener(streamingDecoder);
}
private ServerCallListener startListener(
RpcInvocation invocation, MethodDescriptor methodDescriptor, Invoker<?> invoker) {
+ // Process reliability negotiation
+ java.util.Map<String, Object> attachments = invocation.getObjectAttachments();
+ boolean reliabilityEnabled = attachments.containsKey("tri-reliable-version");
+ String sessionId = null;
+
+ if (reliabilityEnabled) {
+ sessionId = (String) attachments.get("tri-session-id");
+ // Server confirms support
+ org.apache.dubbo.rpc.RpcContext.getContext().setAttachment("tri-reliable-ack", "1.0");
+ org.apache.dubbo.rpc.RpcContext.getContext().setAttachment("tri-session-id", sessionId);
+ }
+
switch (methodDescriptor.getRpcType()) {
case UNARY:
prepareUnaryServerCall();
return new UnaryServerCallListener(invocation, invoker, responseObserver);
case SERVER_STREAM:
prepareStreamServerCall();
- return new ServerStreamServerCallListener(invocation, invoker, responseObserver);
+ return new ServerStreamServerCallListener(
+ invocation, invoker, responseObserver, reliabilityEnabled, sessionId);
case BI_STREAM:
case CLIENT_STREAM:
prepareStreamServerCall();
- return new BiStreamServerCallListener(invocation, invoker, responseObserver);
+ return new BiStreamServerCallListener(
+ invocation, invoker, responseObserver, reliabilityEnabled, sessionId);
default:
throw new IllegalStateException("Can not reach here");
}
@@ -133,15 +155,97 @@ public class GenericHttp2ServerTransportListener extends AbstractServerTransport
}
}
+ private static final ThreadLocal<Long> currentMessageSequence = new ThreadLocal<>();
+
@Override
protected void onMetadataCompletion(Http2Header metadata) {
responseObserver.setResponseEncoder(getContext().getHttpMessageEncoder());
responseObserver.request(1);
+
+ // Process heartbeat first for immediate response
+ processHeartbeatHeaders(metadata);
+
+ // Extract sequence number from headers for reliable streaming
+ String seqHeader = metadata.header("tri-seq");
+ if (seqHeader != null) {
+ try {
+ Long sequenceNumber = Long.parseLong(seqHeader);
+ // Set in ThreadLocal for current message processing
+ currentMessageSequence.set(sequenceNumber);
+ } catch (NumberFormatException e) {
+ // Log and continue - not a critical error
+ }
+ }
+
if (metadata.isEndStream()) {
getStreamingDecoder().close();
}
}
+ private void processHeartbeatHeaders(Http2Header metadata) {
+ // Check for heartbeat from client
+ String heartbeatHeader = metadata.header("tri-heartbeat");
+ if (heartbeatHeader != null) {
+ sendHeartbeatAck(heartbeatHeader);
+ return;
+ }
+
+ // Check for heartbeat ACK from client (for bidirectional heartbeat)
+ String heartbeatAckHeader = metadata.header("tri-heartbeat-ack");
+ if (heartbeatAckHeader != null) {
+ // Server received ACK from client, update last received time if needed
+ // This is useful for bidirectional streaming where server also sends heartbeats
+ LOGGER.debug("Received heartbeat ACK from client: {}", heartbeatAckHeader);
+ }
+ }
+
+ private void sendHeartbeatAck(String originalHeartbeat) {
+ try {
+ // Create HTTP/2 metadata frame for immediate heartbeat ACK
+ DefaultHttpHeaders headers = new DefaultHttpHeaders();
+ headers.set("tri-heartbeat-ack", originalHeartbeat);
+ headers.set("tri-server-time", String.valueOf(System.currentTimeMillis()));
+
+ // Create HTTP/2 metadata frame (HEADERS frame without ending stream)
+ Http2MetadataFrame heartbeatFrame = new Http2MetadataFrame(headers, false);
+
+ // Send immediately via H2StreamChannel for fast response
+ java.util.concurrent.CompletableFuture<Void> future = h2StreamChannel.writeHeader(heartbeatFrame);
+ h2StreamChannel.flush();
+
+ // Also set in response attachments for compatibility
+ if (responseObserver instanceof AttachmentHolder) {
+ AttachmentHolder attachmentHolder = (AttachmentHolder) responseObserver;
+ java.util.Map<String, Object> existingAttachments = attachmentHolder.getResponseAttachments();
+ java.util.Map<String, Object> heartbeatAttachments = existingAttachments != null
+ ? new java.util.HashMap<>(existingAttachments)
+ : new java.util.HashMap<>();
+
+ heartbeatAttachments.put("tri-heartbeat-ack", originalHeartbeat);
+ heartbeatAttachments.put("tri-server-time", String.valueOf(System.currentTimeMillis()));
+ attachmentHolder.setResponseAttachments(heartbeatAttachments);
+ }
+
+ future.whenComplete((result, throwable) -> {
+ if (throwable != null) {
+ LOGGER.debug("Failed to send immediate heartbeat ACK frame", throwable);
+ } else {
+ LOGGER.debug("Sent immediate heartbeat ACK to client: {}", originalHeartbeat);
+ }
+ });
+
+ } catch (Exception e) {
+ LOGGER.warn(LoggerCodeConstants.INTERNAL_ERROR, "", "", "Failed to send heartbeat ACK", e);
+ }
+ }
+
+ Long extractCurrentSequence() {
+ Long sequence = currentMessageSequence.get();
+ // Clear after use to prevent memory leaks
+ currentMessageSequence.remove();
+ return sequence;
+ }
+
@Override
protected void onDataCompletion(Http2InputMessage message) {
if (message.isEndStream()) {
@@ -185,19 +289,55 @@ public class GenericHttp2ServerTransportListener extends AbstractServerTransport
@Override
public void close() {
- responseObserver.close();
+ try {
+ // Clear ThreadLocal to prevent memory leaks
+ currentMessageSequence.remove();
+
+ // Close response observer
+ if (responseObserver != null) {
+ responseObserver.close();
+ }
+
+ // Cancel server call listener if needed
+ if (serverCallListener != null) {
+ try {
+ serverCallListener.onCancel(0); // Graceful cancellation
+ } catch (Exception e) {
+ LOGGER.debug("Error cancelling server call listener during close", e);
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Error during GenericHttp2ServerTransportListener close",
+ e);
+ }
}
private static final class Http2StreamingDecodeListener implements ListeningDecoder.Listener {
private final ServerCallListener serverCallListener;
+ private final GenericHttp2ServerTransportListener transportListener;
- Http2StreamingDecodeListener(ServerCallListener serverCallListener) {
+ Http2StreamingDecodeListener(
+ ServerCallListener serverCallListener, GenericHttp2ServerTransportListener transportListener) {
this.serverCallListener = serverCallListener;
+ this.transportListener = transportListener;
}
@Override
public void onMessage(Object message) {
+ // Pass sequence number to ServerCallListener before processing message
+ Long currentSeq = transportListener.extractCurrentSequence();
+ if (currentSeq != null) {
+ if (serverCallListener instanceof BiStreamServerCallListener) {
+ ((BiStreamServerCallListener) serverCallListener).setCurrentSequence(currentSeq);
+ } else if (serverCallListener instanceof ServerStreamServerCallListener) {
+ ((ServerStreamServerCallListener) serverCallListener).setCurrentSequence(currentSeq);
+ }
+ }
serverCallListener.onMessage(message);
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/http2/Http2ServerStreamObserver.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/http2/Http2ServerStreamObserver.java
index ea959f2634..53df3e0184 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/http2/Http2ServerStreamObserver.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/h12/http2/Http2ServerStreamObserver.java
@@ -16,23 +16,34 @@
*/
package org.apache.dubbo.rpc.protocol.tri.h12.http2;
+import org.apache.dubbo.common.constants.LoggerCodeConstants;
+import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
+import org.apache.dubbo.common.logger.LoggerFactory;
import org.apache.dubbo.remoting.http12.HttpHeaders;
import org.apache.dubbo.remoting.http12.HttpMetadata;
import org.apache.dubbo.remoting.http12.h2.H2StreamChannel;
+import org.apache.dubbo.remoting.http12.h2.Http2MetadataFrame;
import org.apache.dubbo.remoting.http12.h2.Http2ServerChannelObserver;
+import org.apache.dubbo.remoting.http12.netty4.NettyHttpHeaders;
import org.apache.dubbo.rpc.model.FrameworkModel;
import org.apache.dubbo.rpc.protocol.tri.ServerStreamObserver;
import org.apache.dubbo.rpc.protocol.tri.TripleProtocol;
import org.apache.dubbo.rpc.protocol.tri.compressor.Compressor;
import org.apache.dubbo.rpc.protocol.tri.h12.AttachmentHolder;
import org.apache.dubbo.rpc.protocol.tri.h12.CompressibleEncoder;
+import org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext;
import org.apache.dubbo.rpc.protocol.tri.stream.StreamUtils;
import java.util.Map;
+import io.netty.handler.codec.http2.DefaultHttp2Headers;
+
public class Http2ServerStreamObserver extends Http2ServerChannelObserver
implements ServerStreamObserver<Object>, AttachmentHolder {
+ private static final ErrorTypeAwareLogger LOGGER =
+ LoggerFactory.getErrorTypeAwareLogger(Http2ServerStreamObserver.class);
+
private final FrameworkModel frameworkModel;
private Map<String, Object> attachments;
@@ -66,4 +77,43 @@ public class Http2ServerStreamObserver extends Http2ServerChannelObserver
StreamUtils.putHeaders(headers, attachments, TripleProtocol.CONVERT_NO_LOWER_HEADER);
return metadata;
}
+
+ @Override
+ public ReliabilityContext reliability() {
+ // Server-side streams don't support reliability context in this implementation
+ return null;
+ }
+
+ public void sendAckHeaders(Map<String, Object> ackAttachments) {
+ try {
+ DefaultHttp2Headers headers = new DefaultHttp2Headers(false, ackAttachments.size());
+
+ // Convert attachments to HTTP/2 headers
+ for (Map.Entry<String, Object> entry : ackAttachments.entrySet()) {
+ String key = entry.getKey();
+ Object value = entry.getValue();
+ if (value != null) {
+ headers.set(key, value.toString());
+ }
+ }
+
+ // Write headers frame directly to the HTTP/2 stream and flush immediately
+ // This ensures ACK headers are sent immediately without waiting for data
+ H2StreamChannel streamChannel = getHttpChannel();
+ if (streamChannel != null) {
+ // Create and send a HEADERS frame (not end stream)
+ HttpMetadata metadata = new Http2MetadataFrame(new NettyHttpHeaders<>(headers), false);
+
+ // Write and flush to ensure immediate transmission
+ streamChannel.writeHeader(metadata);
+ streamChannel.flush();
+
+ // Debug log to confirm ACK headers were sent
+ LOGGER.debug("Server sent ACK headers and flushed: {}", ackAttachments);
+ }
+ } catch (Exception e) {
+ // Log but don't throw to maintain stream stability
+ LOGGER.warn(LoggerCodeConstants.INTERNAL_ERROR, "", "", "Failed to send ACK headers", e);
+ }
+ }
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/observer/CallStreamObserver.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/observer/CallStreamObserver.java
index d7d74dfb88..3efd8b534f 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/observer/CallStreamObserver.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/observer/CallStreamObserver.java
@@ -18,6 +18,7 @@ package org.apache.dubbo.rpc.protocol.tri.observer;
import org.apache.dubbo.common.stream.StreamObserver;
import org.apache.dubbo.rpc.protocol.tri.compressor.Compressor;
+import org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext;
public interface CallStreamObserver<T> extends StreamObserver<T> {
@@ -48,4 +49,12 @@ public interface CallStreamObserver<T> extends StreamObserver<T> {
* specified.
*/
void disableAutoFlowControl();
+
+ /**
+ * Get the reliability context for this stream.
+ * This provides business observability and control over reliability features.
+ *
+ * @return ReliabilityContext instance, or null if reliability is not enabled
+ */
+ ReliabilityContext reliability();
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/observer/ClientCallToObserverAdapter.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/observer/ClientCallToObserverAdapter.java
index 9f9b054029..9ebd174f7f 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/observer/ClientCallToObserverAdapter.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/observer/ClientCallToObserverAdapter.java
@@ -19,6 +19,7 @@ package org.apache.dubbo.rpc.protocol.tri.observer;
import org.apache.dubbo.rpc.protocol.tri.CancelableStreamObserver;
import org.apache.dubbo.rpc.protocol.tri.ClientStreamObserver;
import org.apache.dubbo.rpc.protocol.tri.call.ClientCall;
+import org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityContext;
public class ClientCallToObserverAdapter<T> extends CancelableStreamObserver<T> implements ClientStreamObserver<T> {
@@ -76,4 +77,13 @@ public class ClientCallToObserverAdapter<T> extends CancelableStreamObserver<T>
public void disableAutoFlowControl() {
call.setAutoRequest(false);
}
+
+ @Override
+ public ReliabilityContext reliability() {
+ // Delegate to the underlying call if it supports reliability
+ if (call instanceof ReliabilityContext) {
+ return (ReliabilityContext) call;
+ }
+ return null;
+ }
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/FileServerSequenceStore.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/FileServerSequenceStore.java
new file mode 100644
index 0000000000..a6d9be2716
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/FileServerSequenceStore.java
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.server;
+
+import org.apache.dubbo.common.constants.LoggerCodeConstants;
+import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * File-based implementation of ServerSequenceStore
+ * 文件实现的服务端序列号存储,支持跨进程重启的持久化
+ */
+public class FileServerSequenceStore implements ServerSequenceStore {
+
+ private static final ErrorTypeAwareLogger LOGGER =
+ LoggerFactory.getErrorTypeAwareLogger(FileServerSequenceStore.class);
+
+ private volatile String currentSessionId;
+ private volatile String currentServiceKey;
+ private volatile Path storeFile;
+ private volatile boolean initialized = false;
+
+ // In-memory cache for performance
+ private final ConcurrentSkipListSet<Long> processedSequences = new ConcurrentSkipListSet<>();
+ private final AtomicLong lastAcknowledgedSequence = new AtomicLong(-1);
+ private final AtomicLong totalSequencesRecorded = new AtomicLong(0);
+ private final AtomicLong cleanupCount = new AtomicLong(0);
+
+ // File operations lock
+ private final ReentrantReadWriteLock fileLock = new ReentrantReadWriteLock();
+
+ // Store directory
+ private static final String DEFAULT_STORE_DIR = System.getProperty("user.home") + "/.dubbo/tri/sequences";
+ private final String storeDirectory;
+
+ public FileServerSequenceStore() {
+ this(DEFAULT_STORE_DIR);
+ }
+
+ public FileServerSequenceStore(String storeDirectory) {
+ this.storeDirectory = storeDirectory;
+ }
+
+ @Override
+ public void init(String sessionId, String serviceKey) throws ServerSequenceException {
+ try {
+ this.currentSessionId = sessionId;
+ this.currentServiceKey = serviceKey;
+
+ // Create store directory if it doesn't exist
+ Path dirPath = Paths.get(storeDirectory);
+ if (!Files.exists(dirPath)) {
+ Files.createDirectories(dirPath);
+ }
+
+ // Create store file path
+ this.storeFile = dirPath.resolve(sessionId + "_" + serviceKey.replace(".", "_") + ".seq");
+
+ // Load existing data if file exists
+ if (Files.exists(storeFile)) {
+ loadFromFile();
+ }
+
+ this.initialized = true;
+ LOGGER.info("Initialized FileServerSequenceStore for session: {} at file: {}", sessionId, storeFile);
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to initialize file store for session: " + sessionId, e);
+ }
+ }
+
+ @Override
+ public void recordSequence(long sequence) throws ServerSequenceException {
+ if (!initialized) {
+ throw new ServerSequenceException("Store not initialized");
+ }
+
+ try {
+ // Check if already processed (in-memory cache)
+ if (processedSequences.contains(sequence)) {
+ return;
+ }
+
+ // Add to in-memory cache
+ if (processedSequences.add(sequence)) {
+ totalSequencesRecorded.incrementAndGet();
+ lastAcknowledgedSequence.updateAndGet(current -> Math.max(current, sequence));
+
+ // Persist to file
+ persistToFile();
+ }
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to record sequence: " + sequence, e);
+ }
+ }
+
+ @Override
+ public long getLastAcknowledgedSequence() throws ServerSequenceException {
+ if (!initialized) {
+ return -1;
+ }
+ return lastAcknowledgedSequence.get();
+ }
+
+ @Override
+ public boolean isSequenceProcessed(long sequence) throws ServerSequenceException {
+ if (!initialized) {
+ return false;
+ }
+
+ // First check in-memory cache
+ if (processedSequences.contains(sequence)) {
+ return true;
+ }
+
+ // If not in cache, check file (this handles case where cache was cleared but file exists)
+ try {
+ fileLock.readLock().lock();
+ if (Files.exists(storeFile)) {
+ try (BufferedReader reader = Files.newBufferedReader(storeFile)) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ try {
+ long seq = Long.parseLong(line.trim());
+ if (seq == sequence) {
+ return true;
+ }
+ } catch (NumberFormatException e) {
+ // Skip invalid lines
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Invalid sequence number in file: " + line);
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to check sequence from file", e);
+ } finally {
+ fileLock.readLock().unlock();
+ }
+
+ return false;
+ }
+
+ @Override
+ public void cleanupSequencesUpTo(long sequence) throws ServerSequenceException {
+ if (!initialized) {
+ return;
+ }
+
+ try {
+ // Remove from in-memory cache
+ if (processedSequences.headSet(sequence).size() > 0) {
+ processedSequences.headSet(sequence).clear();
+ cleanupCount.incrementAndGet();
+ }
+
+ // Rebuild file with remaining sequences
+ persistToFile();
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to cleanup sequences up to: " + sequence, e);
+ }
+ }
+
+ @Override
+ public void cleanupAcknowledged(long sequence) throws ServerSequenceException {
+ if (!initialized) {
+ return;
+ }
+
+ try {
+ // Remove acknowledged sequences from in-memory cache
+ if (processedSequences.headSet(sequence).size() > 0) {
+ int cleanedCount = processedSequences.headSet(sequence).size();
+ processedSequences.headSet(sequence).clear();
+ cleanupCount.incrementAndGet();
+
+ // Update last acknowledged sequence if needed
+ lastAcknowledgedSequence.updateAndGet(current -> Math.max(current, sequence));
+
+ LOGGER.info(
+ "Cleaned up {} acknowledged sequences up to: {} for session: {}",
+ cleanedCount,
+ sequence,
+ currentSessionId);
+ }
+
+ // Rebuild file with remaining sequences (only unacknowledged ones)
+ persistToFile();
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to cleanup acknowledged sequences up to: " + sequence, e);
+ }
+ }
+
+ @Override
+ public StoreStats getStats() throws ServerSequenceException {
+ if (!initialized) {
+ return new StoreStats(0, 0, -1, 0);
+ }
+
+ return new StoreStats(
+ totalSequencesRecorded.get(),
+ processedSequences.size(),
+ lastAcknowledgedSequence.get(),
+ cleanupCount.get());
+ }
+
+ @Override
+ public void close() throws ServerSequenceException {
+ try {
+ if (initialized && storeFile != null) {
+ // Final persist before closing
+ persistToFile();
+
+ // Clear in-memory cache
+ int clearedCount = processedSequences.size();
+ processedSequences.clear();
+
+ LOGGER.info(
+ "Closed FileServerSequenceStore for session: {}, cleared {} sequences",
+ currentSessionId,
+ clearedCount);
+ }
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to close file store for session: " + currentSessionId, e);
+ } finally {
+ initialized = false;
+ }
+ }
+
+ @Override
+ public boolean isHealthy() {
+ if (!initialized || storeFile == null) {
+ return false;
+ }
+
+ // Check if we can write to the store directory
+ try {
+ Path testFile = Paths.get(storeDirectory, "health_test_" + System.currentTimeMillis() + ".tmp");
+ Files.write(testFile, "test".getBytes());
+ Files.deleteIfExists(testFile);
+ return true;
+ } catch (Exception e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "File store health check failed for directory: " + storeDirectory,
+ e);
+ return false;
+ }
+ }
+
+ /**
+ * Persist current state to file
+ */
+ private void persistToFile() throws ServerSequenceException {
+ fileLock.writeLock().lock();
+ try {
+ // Write to temporary file first, then rename for atomicity
+ Path tempFile = Paths.get(storeFile.toString() + ".tmp");
+
+ try (BufferedWriter writer = Files.newBufferedWriter(tempFile)) {
+ for (Long sequence : processedSequences) {
+ writer.write(sequence.toString());
+ writer.newLine();
+ }
+ }
+
+ // Atomic replace
+ Files.move(tempFile, storeFile, java.nio.file.StandardCopyOption.REPLACE_EXISTING);
+
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to persist sequences to file: " + storeFile, e);
+ } finally {
+ fileLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Load existing data from file
+ */
+ private void loadFromFile() throws ServerSequenceException {
+ fileLock.readLock().lock();
+ try {
+ try (BufferedReader reader = Files.newBufferedReader(storeFile)) {
+ String line;
+ long maxSeq = -1;
+ int count = 0;
+
+ while ((line = reader.readLine()) != null) {
+ try {
+ long sequence = Long.parseLong(line.trim());
+ processedSequences.add(sequence);
+ maxSeq = Math.max(maxSeq, sequence);
+ count++;
+ } catch (NumberFormatException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Invalid sequence number in file: " + line + ", skipping");
+ }
+ }
+
+ if (maxSeq >= 0) {
+ lastAcknowledgedSequence.set(maxSeq);
+ }
+ totalSequencesRecorded.set(count);
+
+ LOGGER.info("Loaded {} sequences from file: {}", count, storeFile);
+ }
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to load sequences from file: " + storeFile, e);
+ } finally {
+ fileLock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Get the store file path (for testing and monitoring)
+ */
+ public Path getStoreFilePath() {
+ return storeFile;
+ }
+
+ /**
+ * Get the store directory (for testing and monitoring)
+ */
+ public String getStoreDirectory() {
+ return storeDirectory;
+ }
+
+ /**
+ * Force immediate persistence (for testing)
+ */
+ public void forcePersist() throws ServerSequenceException {
+ persistToFile();
+ }
+
+ /**
+ * Clear all data (for testing)
+ */
+ public void clear() throws ServerSequenceException {
+ fileLock.writeLock().lock();
+ try {
+ processedSequences.clear();
+ lastAcknowledgedSequence.set(-1);
+ totalSequencesRecorded.set(0);
+ cleanupCount.set(0);
+
+ if (storeFile != null) {
+ Files.deleteIfExists(storeFile);
+ }
+
+ LOGGER.info("Cleared all data from FileServerSequenceStore for session: {}", currentSessionId);
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to clear file store", e);
+ } finally {
+ fileLock.writeLock().unlock();
+ }
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/InMemoryServerSequenceStore.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/InMemoryServerSequenceStore.java
new file mode 100644
index 0000000000..0f4255c262
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/InMemoryServerSequenceStore.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.server;
+
+import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * In-memory implementation of ServerSequenceStore
+ * 内存实现的服务端序列号存储,提供基本功能但不支持跨进程重启
+ */
+public class InMemoryServerSequenceStore implements ServerSequenceStore {
+
+ private static final ErrorTypeAwareLogger LOGGER =
+ LoggerFactory.getErrorTypeAwareLogger(InMemoryServerSequenceStore.class);
+
+ // Session-specific storage
+ private final ConcurrentHashMap<String, SessionStorage> sessionStores = new ConcurrentHashMap<>();
+ private volatile String currentSessionId;
+ private volatile String currentServiceKey;
+
+ // Session-specific storage data
+ private static class SessionStorage {
+ private final ConcurrentSkipListSet<Long> processedSequences = new ConcurrentSkipListSet<>();
+ private final AtomicLong lastAcknowledgedSequence = new AtomicLong(-1);
+ private final AtomicLong totalSequencesRecorded = new AtomicLong(0);
+ private final AtomicLong cleanupCount = new AtomicLong(0);
+
+ boolean recordSequence(long sequence) {
+ if (processedSequences.add(sequence)) {
+ totalSequencesRecorded.incrementAndGet();
+ return true;
+ }
+ return false;
+ }
+
+ void updateLastAcknowledged(long sequence) {
+ lastAcknowledgedSequence.updateAndGet(current -> Math.max(current, sequence));
+ }
+
+ void cleanupSequencesUpTo(long sequence) {
+ if (processedSequences.headSet(sequence).size() > 0) {
+ processedSequences.headSet(sequence).clear();
+ cleanupCount.incrementAndGet();
+ }
+ }
+ }
+
+ @Override
+ public void init(String sessionId, String serviceKey) throws ServerSequenceException {
+ this.currentSessionId = sessionId;
+ this.currentServiceKey = serviceKey;
+ sessionStores.putIfAbsent(sessionId, new SessionStorage());
+ LOGGER.info("Initialized InMemoryServerSequenceStore for session: {}", sessionId);
+ }
+
+ @Override
+ public void recordSequence(long sequence) throws ServerSequenceException {
+ try {
+ SessionStorage storage = sessionStores.get(currentSessionId);
+ if (storage == null) {
+ throw new ServerSequenceException("Session not initialized: " + currentSessionId);
+ }
+
+ storage.recordSequence(sequence);
+ storage.updateLastAcknowledged(sequence);
+
+ LOGGER.debug("Recorded sequence: {} for session: {}", sequence, currentSessionId);
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to record sequence: " + sequence, e);
+ }
+ }
+
+ @Override
+ public long getLastAcknowledgedSequence() throws ServerSequenceException {
+ try {
+ SessionStorage storage = sessionStores.get(currentSessionId);
+ if (storage == null) {
+ return -1;
+ }
+ return storage.lastAcknowledgedSequence.get();
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to get last acknowledged sequence", e);
+ }
+ }
+
+ @Override
+ public boolean isSequenceProcessed(long sequence) throws ServerSequenceException {
+ try {
+ SessionStorage storage = sessionStores.get(currentSessionId);
+ if (storage == null) {
+ return false;
+ }
+ return storage.processedSequences.contains(sequence);
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to check sequence processed status", e);
+ }
+ }
+
+ @Override
+ public void cleanupSequencesUpTo(long sequence) throws ServerSequenceException {
+ try {
+ SessionStorage storage = sessionStores.get(currentSessionId);
+ if (storage != null) {
+ storage.cleanupSequencesUpTo(sequence);
+ LOGGER.debug("Cleaned up sequences up to: {} for session: {}", sequence, currentSessionId);
+ }
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to cleanup sequences up to: " + sequence, e);
+ }
+ }
+
+ @Override
+ public void cleanupAcknowledged(long sequence) throws ServerSequenceException {
+ try {
+ SessionStorage storage = sessionStores.get(currentSessionId);
+ if (storage != null) {
+ // Clean up sequences up to the acknowledged sequence
+ storage.cleanupSequencesUpTo(sequence);
+
+ // Update last acknowledged sequence if needed
+ storage.updateLastAcknowledged(sequence);
+
+ LOGGER.debug("Cleaned up acknowledged sequences up to: {} for session: {}", sequence, currentSessionId);
+ }
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to cleanup acknowledged sequences up to: " + sequence, e);
+ }
+ }
+
+ @Override
+ public StoreStats getStats() throws ServerSequenceException {
+ try {
+ SessionStorage storage = sessionStores.get(currentSessionId);
+ if (storage == null) {
+ return new StoreStats(0, 0, -1, 0);
+ }
+
+ return new StoreStats(
+ storage.totalSequencesRecorded.get(),
+ storage.processedSequences.size(),
+ storage.lastAcknowledgedSequence.get(),
+ storage.cleanupCount.get());
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to get store stats", e);
+ }
+ }
+
+ @Override
+ public void close() throws ServerSequenceException {
+ try {
+ SessionStorage storage = sessionStores.remove(currentSessionId);
+ if (storage != null) {
+ LOGGER.info(
+ "Closed InMemoryServerSequenceStore for session: {}, cleaned {} sequences",
+ currentSessionId,
+ storage.processedSequences.size());
+ }
+ } catch (Exception e) {
+ throw new ServerSequenceException("Failed to close store for session: " + currentSessionId, e);
+ }
+ }
+
+ @Override
+ public boolean isHealthy() {
+ return true; // In-memory store is always healthy
+ }
+
+ /**
+ * Get the number of sessions in the store (for monitoring)
+ */
+ public int getSessionCount() {
+ return sessionStores.size();
+ }
+
+ /**
+ * Get the number of processed sequences for a session (for monitoring)
+ */
+ public int getProcessedSequenceCount(String sessionId) {
+ SessionStorage storage = sessionStores.get(sessionId);
+ return storage != null ? storage.processedSequences.size() : 0;
+ }
+
+ /**
+ * Clear all data (for testing)
+ */
+ public void clear() {
+ sessionStores.clear();
+ LOGGER.info("Cleared all data from InMemoryServerSequenceStore");
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/ServerSequenceException.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/ServerSequenceException.java
new file mode 100644
index 0000000000..397c730f49
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/ServerSequenceException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.server;
+
+/**
+ * Exception thrown by ServerSequenceStore operations
+ * 服务端序列号存储操作异常
+ */
+public class ServerSequenceException extends Exception {
+
+ public ServerSequenceException(String message) {
+ super(message);
+ }
+
+ public ServerSequenceException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ServerSequenceException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/ServerSequenceStore.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/ServerSequenceStore.java
new file mode 100644
index 0000000000..510e5bbd53
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/server/ServerSequenceStore.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.server;
+
+import org.apache.dubbo.common.extension.ExtensionScope;
+import org.apache.dubbo.common.extension.SPI;
+
+/**
+ * Service-side sequence number state storage interface for reliable streaming
+ * 服务端序列号状态存储接口,用于实现重启后的幂等性保证
+ */
+@SPI(scope = ExtensionScope.FRAMEWORK)
+public interface ServerSequenceStore {
+
+ /**
+ * Initialize the store for a session
+ * 初始化会话的存储
+ *
+ * @param sessionId session identifier
+ * @param serviceKey service key
+ * @throws ServerSequenceException if initialization fails
+ */
+ void init(String sessionId, String serviceKey) throws ServerSequenceException;
+
+ /**
+ * Record a processed sequence number
+ * 记录已处理的序列号
+ *
+ * @param sequence sequence number
+ * @throws ServerSequenceException if storage fails
+ */
+ void recordSequence(long sequence) throws ServerSequenceException;
+
+ /**
+ * Get the last acknowledged sequence number
+ * 获取最后确认的序列号
+ *
+ * @return last acknowledged sequence number, or -1 if not found
+ * @throws ServerSequenceException if retrieval fails
+ */
+ long getLastAcknowledgedSequence() throws ServerSequenceException;
+
+ /**
+ * Check if a sequence number has been processed
+ * 检查序列号是否已处理过
+ *
+ * @param sequence sequence number to check
+ * @return true if processed, false otherwise
+ * @throws ServerSequenceException if check fails
+ */
+ boolean isSequenceProcessed(long sequence) throws ServerSequenceException;
+
+ /**
+ * Clean up sequences up to the specified sequence number (inclusive)
+ * 清理指定序列号之前的记录(包含该序列号)
+ *
+ * @param sequence sequence number to clean up to
+ * @throws ServerSequenceException if cleanup fails
+ */
+ void cleanupSequencesUpTo(long sequence) throws ServerSequenceException;
+
+ /**
+ * Get statistics about the store
+ * 获取存储统计信息
+ *
+ * @return store statistics
+ * @throws ServerSequenceException if retrieval fails
+ */
+ StoreStats getStats() throws ServerSequenceException;
+
+ /**
+ * Clean up acknowledged sequences up to the specified sequence number (inclusive)
+ * 清理已确认的序列号,支持基于ACK的增量清理
+ *
+ * @param sequence sequence number to clean up to
+ * @throws ServerSequenceException if cleanup fails
+ */
+ void cleanupAcknowledged(long sequence) throws ServerSequenceException;
+
+ /**
+ * Close the store and cleanup resources
+ * 关闭存储并清理资源
+ *
+ * @throws ServerSequenceException if close fails
+ */
+ void close() throws ServerSequenceException;
+
+ /**
+ * Check if the store is healthy
+ * 检查存储是否健康
+ *
+ * @return true if healthy, false otherwise
+ */
+ boolean isHealthy();
+
+ /**
+ * Store statistics
+ * 存储统计信息
+ */
+ class StoreStats {
+ private final long totalSequencesRecorded;
+ private final long currentSequenceCount;
+ private final long lastAcknowledgedSequence;
+ private final long cleanupCount;
+
+ public StoreStats(
+ long totalSequencesRecorded,
+ long currentSequenceCount,
+ long lastAcknowledgedSequence,
+ long cleanupCount) {
+ this.totalSequencesRecorded = totalSequencesRecorded;
+ this.currentSequenceCount = currentSequenceCount;
+ this.lastAcknowledgedSequence = lastAcknowledgedSequence;
+ this.cleanupCount = cleanupCount;
+ }
+
+ public long getTotalSequencesRecorded() {
+ return totalSequencesRecorded;
+ }
+
+ public long getCurrentSequenceCount() {
+ return currentSequenceCount;
+ }
+
+ public long getLastAcknowledgedSequence() {
+ return lastAcknowledgedSequence;
+ }
+
+ public long getCleanupCount() {
+ return cleanupCount;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "StoreStats{total=%d, current=%d, lastAcked=%d, cleanups=%d}",
+ totalSequencesRecorded, currentSequenceCount, lastAcknowledgedSequence, cleanupCount);
+ }
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/FilePendingMessageStore.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/FilePendingMessageStore.java
new file mode 100644
index 0000000000..b1ef540841
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/FilePendingMessageStore.java
@@ -0,0 +1,663 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.store;
+
+import org.apache.dubbo.common.constants.LoggerCodeConstants;
+import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * File-based implementation of PendingMessageStore.
+ * Provides persistence across process restarts using append-only log files.
+ */
+public class FilePendingMessageStore implements PendingMessageStore {
+
+ private static final ErrorTypeAwareLogger LOGGER =
+ LoggerFactory.getErrorTypeAwareLogger(FilePendingMessageStore.class);
+
+ private static final String FILE_SUFFIX = ".pmsg";
+ private static final String TEMP_FILE_SUFFIX = ".tmp";
+ private static final int MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+ private static final long DEFAULT_RETENTION_TIME = 24 * 60 * 60 * 1000; // 24 hours
+
+ // ACK cleanup constants
+ private static final int CLEANUP_BATCH_SIZE = 100; // Clean up after every 100 ACKs
+ private static final int CLEANUP_INTERVAL = 1000; // Clean up every 1000 sequences
+ private static final double COMPRESSION_THRESHOLD = 0.3; // Compress when 30% of messages are acknowledged
+
+ private final String basePath;
+ private final long maxFileSize;
+ private final long retentionTime;
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ private final AtomicBoolean healthy = new AtomicBoolean(true);
+
+ private SessionMetadata currentMetadata;
+ private Path currentFile;
+ private DataOutputStream outputStream;
+
+ // ACK tracking fields
+ private final AtomicLong lastAcknowledgedSequence = new AtomicLong(-1);
+ private final AtomicLong totalMessagesRecorded = new AtomicLong(0);
+ private final AtomicLong cleanupCount = new AtomicLong(0);
+
+ public FilePendingMessageStore() {
+ this("/tmp/dubbo/reliable", MAX_FILE_SIZE, DEFAULT_RETENTION_TIME);
+ }
+
+ public FilePendingMessageStore(String basePath, long maxFileSize, long retentionTime) {
+ this.basePath = basePath;
+ this.maxFileSize = maxFileSize;
+ this.retentionTime = retentionTime;
+ }
+
+ @Override
+ public void init(SessionMetadata metadata) throws StoreException {
+ lock.writeLock().lock();
+ try {
+ this.currentMetadata = metadata;
+
+ // Create base directory if it doesn't exist
+ Path baseDir = Paths.get(basePath);
+ if (!Files.exists(baseDir)) {
+ Files.createDirectories(baseDir);
+ }
+
+ // Create session-specific directory
+ Path sessionDir = baseDir.resolve(metadata.getSessionId());
+ if (!Files.exists(sessionDir)) {
+ Files.createDirectories(sessionDir);
+ }
+
+ // Initialize current file
+ currentFile = sessionDir.resolve(System.currentTimeMillis() + FILE_SUFFIX);
+ outputStream =
+ new DataOutputStream(new BufferedOutputStream(new FileOutputStream(currentFile.toFile(), true)));
+
+ // Clean up old files
+ cleanupOldFiles(sessionDir);
+
+ LOGGER.info(
+ "Initialized FilePendingMessageStore for session: {} at path: {}",
+ metadata.getSessionId(),
+ currentFile);
+ } catch (Exception e) {
+ healthy.set(false);
+ throw new StoreException("Failed to initialize store for session: " + metadata.getSessionId(), e);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public void put(PendingMessage message) throws StoreException {
+ if (!healthy.get()) {
+ throw new StoreException("Store is not healthy");
+ }
+
+ lock.writeLock().lock();
+ try {
+ // Check if we need to rotate file
+ if (Files.size(currentFile) > maxFileSize) {
+ rotateFile();
+ }
+
+ // Write message to file
+ writeMessage(message);
+ totalMessagesRecorded.incrementAndGet();
+
+ LOGGER.debug(
+ "Stored message with sequence: {} for session: {}", message.getSequence(), message.getSessionId());
+ } catch (Exception e) {
+ healthy.set(false);
+ throw new StoreException("Failed to store message: " + message.getSequence(), e);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public void ack(long sequence) throws StoreException {
+ if (currentMetadata == null) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Store not initialized, cannot ack sequence: " + sequence);
+ return;
+ }
+
+ try {
+ // Update the last acknowledged sequence
+ long previousAckedSeq = lastAcknowledgedSequence.getAndSet(sequence);
+
+ // Only trigger cleanup if this is a new acknowledgment
+ if (sequence > previousAckedSeq) {
+ // For efficiency, we batch cleanup operations
+ // Only trigger file cleanup when we have accumulated enough acknowledged messages
+ long acknowledgedCount = sequence - previousAckedSeq;
+ if (acknowledgedCount >= CLEANUP_BATCH_SIZE || sequence % CLEANUP_INTERVAL == 0) {
+ cleanupAcknowledgedMessages(sequence);
+ }
+ }
+
+ LOGGER.debug(
+ "Acknowledged message with sequence: {} for session: {}", sequence, currentMetadata.getSessionId());
+ } catch (Exception e) {
+ throw new StoreException("Failed to acknowledge sequence: " + sequence, e);
+ }
+ }
+
+ @Override
+ public void remove(long sequence) throws StoreException {
+ // For append-only log, we don't remove immediately
+ // Cleanup happens during load and file rotation
+ LOGGER.debug("Removed message with sequence: {} for session: {}", sequence, currentMetadata.getSessionId());
+ }
+
+ @Override
+ public List<PendingMessage> load(String sessionId) throws StoreException {
+ lock.readLock().lock();
+ try {
+ List<PendingMessage> messages = new ArrayList<>();
+ Path sessionDir = Paths.get(basePath).resolve(sessionId);
+
+ if (!Files.exists(sessionDir)) {
+ LOGGER.debug("No session directory found for session: {}", sessionId);
+ return messages;
+ }
+
+ // Read all files for the session
+ List<PendingMessage> fileMessages = Files.list(sessionDir)
+ .filter(path -> path.toString().endsWith(FILE_SUFFIX))
+ .flatMap(path -> {
+ try {
+ return readMessagesFromFile(path).stream();
+ } catch (Exception e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to read messages from file: " + path,
+ e);
+ return Stream.empty();
+ }
+ })
+ .collect(Collectors.toList());
+ messages.addAll(fileMessages);
+
+ // Deduplicate by keeping the latest version of each sequence
+ messages = deduplicateMessages(messages);
+
+ LOGGER.info("Loaded {} unique messages for session: {}", messages.size(), sessionId);
+ return messages;
+ } catch (Exception e) {
+ throw new StoreException("Failed to load messages for session: " + sessionId, e);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void close(SessionMetadata metadata) throws StoreException {
+ lock.writeLock().lock();
+ try {
+ if (outputStream != null) {
+ outputStream.close();
+ outputStream = null;
+ }
+
+ LOGGER.info("Closed FilePendingMessageStore for session: {}", metadata.getSessionId());
+ } catch (Exception e) {
+ throw new StoreException("Failed to close store for session: " + metadata.getSessionId(), e);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public boolean isHealthy() {
+ return healthy.get();
+ }
+
+ private void writeMessage(PendingMessage message) throws IOException {
+ writeMessageToStream(outputStream, message);
+ }
+
+ private void writeMessageToStream(DataOutputStream out, PendingMessage message) throws IOException {
+ // Write message format (same as existing format):
+ // [sequence][messageLength][message][compressFlag][sentTime][retryCount][firstSendTime][nextRetryDelay][sessionIdLength][sessionId]
+
+ byte[] messageData = message.getMessage();
+ byte[] sessionIdData = message.getSessionId().getBytes();
+
+ out.writeLong(message.getSequence());
+ out.writeInt(messageData.length);
+ out.write(messageData);
+ out.writeInt(message.getCompressFlag());
+ out.writeLong(message.getSentTime());
+ out.writeInt(message.getRetryCount());
+ out.writeLong(message.getFirstSendTime());
+ out.writeLong(message.getNextRetryDelay());
+ out.writeInt(sessionIdData.length);
+ out.write(sessionIdData);
+ }
+
+ private List<PendingMessage> readMessagesFromFile(Path path) throws IOException {
+ List<PendingMessage> messages = new ArrayList<>();
+
+ try (DataInputStream inputStream = new DataInputStream(
+ new BufferedInputStream(Files.newInputStream(path.toFile().toPath())))) {
+
+ while (inputStream.available() > 0) {
+ try {
+ long sequence = inputStream.readLong();
+ int messageLength = inputStream.readInt();
+ byte[] messageData = new byte[messageLength];
+ inputStream.readFully(messageData);
+ int compressFlag = inputStream.readInt();
+ long sentTime = inputStream.readLong();
+ int retryCount = inputStream.readInt();
+ long firstSendTime = inputStream.readLong();
+ long nextRetryDelay = inputStream.readLong();
+ int sessionIdLength = inputStream.readInt();
+ byte[] sessionIdData = new byte[sessionIdLength];
+ inputStream.readFully(sessionIdData);
+ String sessionId = new String(sessionIdData);
+
+ PendingMessage message = new PendingMessage(
+ sequence,
+ messageData,
+ compressFlag,
+ sentTime,
+ retryCount,
+ firstSendTime,
+ nextRetryDelay,
+ sessionId);
+
+ messages.add(message);
+ } catch (EOFException e) {
+ // End of file reached
+ break;
+ }
+ }
+ }
+
+ return messages;
+ }
+
+ private List<PendingMessage> deduplicateMessages(List<PendingMessage> messages) {
+ // Simple deduplication: keep the latest version of each sequence
+ Map<Long, PendingMessage> latestMessages = new HashMap<>();
+
+ for (PendingMessage message : messages) {
+ PendingMessage existing = latestMessages.get(message.getSequence());
+ if (existing == null || message.getSentTime() > existing.getSentTime()) {
+ latestMessages.put(message.getSequence(), message);
+ }
+ }
+
+ return new ArrayList<>(latestMessages.values());
+ }
+
+ private void rotateFile() throws IOException {
+ lock.writeLock().lock();
+ try {
+ if (outputStream != null) {
+ outputStream.close();
+ }
+
+ // Create new file
+ Path sessionDir = Paths.get(basePath).resolve(currentMetadata.getSessionId());
+ currentFile = sessionDir.resolve(System.currentTimeMillis() + FILE_SUFFIX);
+ outputStream =
+ new DataOutputStream(new BufferedOutputStream(new FileOutputStream(currentFile.toFile(), true)));
+
+ LOGGER.info("Rotated to new file: {} for session: {}", currentFile, currentMetadata.getSessionId());
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void cleanupOldFiles(Path sessionDir) throws IOException {
+ long cutoffTime = System.currentTimeMillis() - retentionTime;
+
+ Files.list(sessionDir)
+ .filter(path -> path.toString().endsWith(FILE_SUFFIX))
+ .filter(path -> {
+ try {
+ return Files.getLastModifiedTime(path).toMillis() < cutoffTime;
+ } catch (IOException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to get last modified time for file: " + path,
+ e);
+ return false;
+ }
+ })
+ .forEach(path -> {
+ try {
+ Files.deleteIfExists(path);
+ LOGGER.debug("Deleted old file: {}", path);
+ } catch (IOException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR, "", "", "Failed to delete old file: " + path, e);
+ }
+ });
+ }
+
+ /**
+ * Clean up acknowledged messages from the store
+ * 清理已确认的消息
+ */
+ private void cleanupAcknowledgedMessages(long ackedSequence) throws StoreException {
+ if (currentMetadata == null) {
+ return;
+ }
+
+ lock.writeLock().lock();
+ try {
+ Path sessionDir = Paths.get(basePath).resolve(currentMetadata.getSessionId());
+ if (!Files.exists(sessionDir)) {
+ return;
+ }
+
+ // Calculate the ratio of acknowledged messages
+ long totalMsgs = totalMessagesRecorded.get();
+ if (totalMsgs == 0) {
+ return;
+ }
+
+ double ackRatio = (double) ackedSequence / totalMsgs;
+
+ // Only perform cleanup if we have enough acknowledged messages
+ if (ackRatio >= COMPRESSION_THRESHOLD) {
+ LOGGER.info(
+ "Starting cleanup for session: {}, acked: {}, total: {}, ratio: {}",
+ currentMetadata.getSessionId(),
+ ackedSequence,
+ totalMsgs,
+ ackRatio);
+
+ // Create a new compressed file containing only unacknowledged messages
+ Path compressedFile = sessionDir.resolve("compressed_" + System.currentTimeMillis() + FILE_SUFFIX);
+
+ try (DataOutputStream compressedOut =
+ new DataOutputStream(new BufferedOutputStream(new FileOutputStream(compressedFile.toFile())))) {
+
+ // Read all files and filter out acknowledged messages
+ List<Path> messageFiles = Files.list(sessionDir)
+ .filter(path -> path.toString().endsWith(FILE_SUFFIX))
+ .sorted()
+ .collect(Collectors.toList());
+
+ long unacknowledgedCount = 0;
+ long acknowledgedCount = 0;
+
+ for (Path file : messageFiles) {
+ try {
+ // Use existing readMessagesFromFile method to read with correct format
+ List<PendingMessage> messages = readMessagesFromFile(file);
+ for (PendingMessage message : messages) {
+ if (message.getSequence() > ackedSequence) {
+ // Write unacknowledged message to compressed file using existing format
+ writeMessageToStream(compressedOut, message);
+ unacknowledgedCount++;
+ } else {
+ acknowledgedCount++;
+ }
+ }
+ } catch (IOException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to read messages from file during cleanup: " + file,
+ e);
+ }
+ }
+
+ // Replace old files with compressed file
+ if (unacknowledgedCount > 0 || acknowledgedCount > 0) {
+ // Delete old files
+ for (Path file : messageFiles) {
+ try {
+ Files.deleteIfExists(file);
+ } catch (IOException e) {
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to delete old file during cleanup: " + file,
+ e);
+ }
+ }
+
+ // Rename compressed file to be the new current file
+ Path newCurrentFile = sessionDir.resolve(System.currentTimeMillis() + FILE_SUFFIX);
+ Files.move(compressedFile, newCurrentFile);
+
+ // Update current file reference
+ if (outputStream != null) {
+ outputStream.close();
+ }
+ currentFile = newCurrentFile;
+ outputStream = new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(currentFile.toFile(), true)));
+
+ cleanupCount.incrementAndGet();
+
+ LOGGER.info(
+ "Cleanup completed for session: {}, removed: {}, kept: {}",
+ currentMetadata.getSessionId(),
+ acknowledgedCount,
+ unacknowledgedCount);
+ } else {
+ // No messages found, delete the empty compressed file
+ Files.deleteIfExists(compressedFile);
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.error(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to cleanup acknowledged messages up to: " + ackedSequence + " for session: "
+ + currentMetadata.getSessionId(),
+ e);
+ throw new StoreException("Failed to cleanup acknowledged messages", e);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public SessionMetadata loadMetadata(String sessionId) throws StoreException {
+ lock.readLock().lock();
+ try {
+ Path baseDir = Paths.get(basePath);
+ Path sessionDir = baseDir.resolve(sessionId);
+ Path metadataFile = sessionDir.resolve("metadata.json");
+
+ if (!Files.exists(metadataFile)) {
+ LOGGER.debug("No metadata file found for session: {}", sessionId);
+ return null;
+ }
+
+ // Read and parse metadata file
+ String jsonContent = new String(Files.readAllBytes(metadataFile), StandardCharsets.UTF_8);
+ return parseMetadataJson(jsonContent);
+
+ } catch (Exception e) {
+ LOGGER.error(
+ LoggerCodeConstants.INTERNAL_ERROR, "", "", "Failed to load metadata for session: " + sessionId, e);
+ // Return null instead of throwing to allow graceful fallback
+ return null;
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateMetadata(SessionMetadata metadata) throws StoreException {
+ lock.writeLock().lock();
+ try {
+ Path baseDir = Paths.get(basePath);
+ Path sessionDir = baseDir.resolve(metadata.getSessionId());
+
+ if (!Files.exists(sessionDir)) {
+ Files.createDirectories(sessionDir);
+ }
+
+ Path metadataFile = sessionDir.resolve("metadata.json");
+ Path tempFile = sessionDir.resolve("metadata.json.tmp");
+
+ // Write to temp file first for atomic update
+ String jsonContent = toMetadataJson(metadata);
+ Files.write(
+ tempFile,
+ jsonContent.getBytes(StandardCharsets.UTF_8),
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.SYNC);
+
+ // Atomic rename
+ Files.move(tempFile, metadataFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+
+ LOGGER.debug("Updated metadata for session: {}", metadata.getSessionId());
+
+ } catch (Exception e) {
+ LOGGER.error(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to update metadata for session: " + metadata.getSessionId(),
+ e);
+ throw new StoreException("Failed to update metadata", e);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Convert SessionMetadata to JSON format
+ */
+ private String toMetadataJson(SessionMetadata metadata) {
+ StringBuilder json = new StringBuilder();
+ json.append("{\n");
+ json.append(" \"sessionId\": \"").append(metadata.getSessionId()).append("\",\n");
+ json.append(" \"serviceKey\": \"").append(metadata.getServiceKey()).append("\",\n");
+ json.append(" \"createTime\": ").append(metadata.getCreateTime()).append(",\n");
+ json.append(" \"lastAckedSeq\": ").append(metadata.getLastAckedSeq()).append(",\n");
+ json.append(" \"currentSeq\": ").append(metadata.getCurrentSeq()).append(",\n");
+ json.append(" \"lastUpdateTime\": ")
+ .append(metadata.getLastUpdateTime())
+ .append("\n");
+ json.append("}");
+ return json.toString();
+ }
+
+ /**
+ * Parse JSON to SessionMetadata
+ */
+ private SessionMetadata parseMetadataJson(String jsonContent) {
+ // Simple JSON parsing without external dependencies
+ try {
+ String sessionId = extractJsonString(jsonContent, "sessionId");
+ String serviceKey = extractJsonString(jsonContent, "serviceKey");
+ long createTime = extractJsonLong(jsonContent, "createTime");
+ long lastAckedSeq = extractJsonLong(jsonContent, "lastAckedSeq");
+ long currentSeq = extractJsonLong(jsonContent, "currentSeq");
+ long lastUpdateTime = extractJsonLong(jsonContent, "lastUpdateTime");
+
+ // Reconstruct URL and config from currentMetadata if available
+ if (currentMetadata != null) {
+ return new SessionMetadata(
+ sessionId,
+ serviceKey,
+ createTime,
+ currentMetadata.getUrl(),
+ currentMetadata.getConfig(),
+ lastAckedSeq,
+ currentSeq,
+ lastUpdateTime);
+ } else {
+ // If no currentMetadata, return partial metadata (will be enriched later)
+ LOGGER.warn(
+ LoggerCodeConstants.INTERNAL_ERROR,
+ "",
+ "",
+ "No current metadata available, returning partial metadata for session: " + sessionId);
+ return null;
+ }
+ } catch (Exception e) {
+ LOGGER.error(LoggerCodeConstants.INTERNAL_ERROR, "", "", "Failed to parse metadata JSON", e);
+ return null;
+ }
+ }
+
+ private String extractJsonString(String json, String key) {
+ String searchKey = "\"" + key + "\": \"";
+ int startIndex = json.indexOf(searchKey);
+ if (startIndex == -1) {
+ return "";
+ }
+ startIndex += searchKey.length();
+ int endIndex = json.indexOf("\"", startIndex);
+ return json.substring(startIndex, endIndex);
+ }
+
+ private long extractJsonLong(String json, String key) {
+ String searchKey = "\"" + key + "\": ";
+ int startIndex = json.indexOf(searchKey);
+ if (startIndex == -1) {
+ return 0L;
+ }
+ startIndex += searchKey.length();
+ int endIndex = json.indexOf(",", startIndex);
+ if (endIndex == -1) {
+ endIndex = json.indexOf("\n", startIndex);
+ }
+ String value = json.substring(startIndex, endIndex).trim();
+ return Long.parseLong(value);
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/InMemoryPendingMessageStore.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/InMemoryPendingMessageStore.java
new file mode 100644
index 0000000000..3eb292ed87
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/InMemoryPendingMessageStore.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.store;
+
+import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * In-memory implementation of PendingMessageStore.
+ * This is the default implementation that provides no persistence across restarts.
+ */
+public class InMemoryPendingMessageStore implements PendingMessageStore {
+
+ private static final ErrorTypeAwareLogger LOGGER =
+ LoggerFactory.getErrorTypeAwareLogger(InMemoryPendingMessageStore.class);
+
+ // sessionId -> sequence -> PendingMessage
+ private final Map<String, Map<Long, PendingMessage>> sessionStore = new ConcurrentHashMap<>();
+
+ private SessionMetadata currentMetadata;
+
+ @Override
+ public void init(SessionMetadata metadata) throws StoreException {
+ this.currentMetadata = metadata;
+ sessionStore.putIfAbsent(metadata.getSessionId(), new ConcurrentHashMap<>());
+ LOGGER.info("Initialized InMemoryPendingMessageStore for session: {}", metadata.getSessionId());
+ }
+
+ @Override
+ public void put(PendingMessage message) throws StoreException {
+ try {
+ Map<Long, PendingMessage> sessionMessages = sessionStore.get(message.getSessionId());
+ if (sessionMessages == null) {
+ throw new StoreException("Session not initialized: " + message.getSessionId());
+ }
+
+ sessionMessages.put(message.getSequence(), message);
+ LOGGER.debug(
+ "Stored message with sequence: {} for session: {}", message.getSequence(), message.getSessionId());
+ } catch (Exception e) {
+ throw new StoreException("Failed to store message: " + message.getSequence(), e);
+ }
+ }
+
+ @Override
+ public void ack(long sequence) throws StoreException {
+ try {
+ if (currentMetadata == null) {
+ throw new StoreException("Store not initialized");
+ }
+
+ Map<Long, PendingMessage> sessionMessages = sessionStore.get(currentMetadata.getSessionId());
+ if (sessionMessages != null) {
+ // Remove all messages with sequence <= acknowledged sequence
+ int removedCount = 0;
+ Iterator<Map.Entry<Long, PendingMessage>> iterator =
+ sessionMessages.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<Long, PendingMessage> entry = iterator.next();
+ if (entry.getKey() <= sequence) {
+ iterator.remove();
+ removedCount++;
+ }
+ }
+ LOGGER.debug(
+ "Acknowledged {} messages up to sequence: {} for session: {}",
+ removedCount,
+ sequence,
+ currentMetadata.getSessionId());
+ }
+ } catch (Exception e) {
+ throw new StoreException("Failed to ack message: " + sequence, e);
+ }
+ }
+
+ @Override
+ public void remove(long sequence) throws StoreException {
+ try {
+ if (currentMetadata == null) {
+ throw new StoreException("Store not initialized");
+ }
+
+ Map<Long, PendingMessage> sessionMessages = sessionStore.get(currentMetadata.getSessionId());
+ if (sessionMessages != null) {
+ sessionMessages.remove(sequence);
+ LOGGER.debug(
+ "Removed message with sequence: {} for session: {}", sequence, currentMetadata.getSessionId());
+ }
+ } catch (Exception e) {
+ throw new StoreException("Failed to remove message: " + sequence, e);
+ }
+ }
+
+ @Override
+ public List<PendingMessage> load(String sessionId) throws StoreException {
+ try {
+ Map<Long, PendingMessage> sessionMessages = sessionStore.get(sessionId);
+ if (sessionMessages == null) {
+ LOGGER.debug("No messages found for session: {}", sessionId);
+ return new ArrayList<>();
+ }
+
+ List<PendingMessage> messages = new ArrayList<>(sessionMessages.values());
+ LOGGER.info("Loaded {} messages for session: {}", messages.size(), sessionId);
+ return messages;
+ } catch (Exception e) {
+ throw new StoreException("Failed to load messages for session: " + sessionId, e);
+ }
+ }
+
+ @Override
+ public void close(SessionMetadata metadata) throws StoreException {
+ try {
+ Map<Long, PendingMessage> sessionMessages = sessionStore.remove(metadata.getSessionId());
+ if (sessionMessages != null) {
+ LOGGER.info(
+ "Closed InMemoryPendingMessageStore for session: {}, removed {} messages",
+ metadata.getSessionId(),
+ sessionMessages.size());
+ }
+ } catch (Exception e) {
+ throw new StoreException("Failed to close store for session: " + metadata.getSessionId(), e);
+ }
+ }
+
+ @Override
+ public boolean isHealthy() {
+ return true; // In-memory store is always healthy
+ }
+
+ @Override
+ public SessionMetadata loadMetadata(String sessionId) throws StoreException {
+ // In-memory store does not persist metadata across process restarts
+ // Return null to indicate no persisted metadata available
+ return null;
+ }
+
+ @Override
+ public void updateMetadata(SessionMetadata metadata) throws StoreException {
+ // In-memory store does not persist metadata
+ // Update current metadata in memory for current session
+ this.currentMetadata = metadata;
+ LOGGER.debug("Updated in-memory metadata for session: {}", metadata.getSessionId());
+ }
+
+ /**
+ * Get the number of pending messages for a session
+ */
+ public int getPendingCount(String sessionId) {
+ Map<Long, PendingMessage> sessionMessages = sessionStore.get(sessionId);
+ return sessionMessages != null ? sessionMessages.size() : 0;
+ }
+
+ /**
+ * Get the total number of sessions
+ */
+ public int getSessionCount() {
+ return sessionStore.size();
+ }
+
+ /**
+ * Clear all data (for testing)
+ */
+ public void clear() {
+ sessionStore.clear();
+ LOGGER.info("Cleared all data from InMemoryPendingMessageStore");
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/PendingMessage.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/PendingMessage.java
new file mode 100644
index 0000000000..3e8de8c457
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/PendingMessage.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.store;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Pending message for reliable streaming with persistence support
+ */
+public class PendingMessage implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final long sequence;
+ private final byte[] message;
+ private final int compressFlag;
+ private final long sentTime;
+ private final int retryCount;
+ private final long firstSendTime;
+ private final long nextRetryDelay;
+ private final String sessionId;
+
+ public PendingMessage(
+ long sequence,
+ byte[] message,
+ int compressFlag,
+ long sentTime,
+ int retryCount,
+ long firstSendTime,
+ long nextRetryDelay,
+ String sessionId) {
+ this.sequence = sequence;
+ this.message = Objects.requireNonNull(message, "message cannot be null");
+ this.compressFlag = compressFlag;
+ this.sentTime = sentTime;
+ this.retryCount = retryCount;
+ this.firstSendTime = firstSendTime;
+ this.nextRetryDelay = nextRetryDelay;
+ this.sessionId = Objects.requireNonNull(sessionId, "sessionId cannot be null");
+ }
+
+ // Constructor for compatibility with existing code
+ public PendingMessage(long sequence, byte[] message, int compressFlag, long sentTime, int retryCount) {
+ this(sequence, message, compressFlag, sentTime, retryCount, sentTime, 100, "unknown");
+ }
+
+ public long getSequence() {
+ return sequence;
+ }
+
+ public byte[] getMessage() {
+ return message;
+ }
+
+ public int getCompressFlag() {
+ return compressFlag;
+ }
+
+ public long getSentTime() {
+ return sentTime;
+ }
+
+ public int getRetryCount() {
+ return retryCount;
+ }
+
+ public long getFirstSendTime() {
+ return firstSendTime;
+ }
+
+ public long getNextRetryDelay() {
+ return nextRetryDelay;
+ }
+
+ public String getSessionId() {
+ return sessionId;
+ }
+
+ public PendingMessage withRetryCount(int newRetryCount) {
+ return new PendingMessage(
+ sequence, message, compressFlag, sentTime, newRetryCount, firstSendTime, nextRetryDelay, sessionId);
+ }
+
+ public PendingMessage withNextRetryDelay(long newNextRetryDelay) {
+ return new PendingMessage(
+ sequence, message, compressFlag, sentTime, retryCount, firstSendTime, newNextRetryDelay, sessionId);
+ }
+
+ public PendingMessage withSessionId(String newSessionId) {
+ return new PendingMessage(
+ sequence, message, compressFlag, sentTime, retryCount, firstSendTime, nextRetryDelay, newSessionId);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ PendingMessage that = (PendingMessage) o;
+ return sequence == that.sequence
+ && compressFlag == that.compressFlag
+ && sentTime == that.sentTime
+ && retryCount == that.retryCount
+ && firstSendTime == that.firstSendTime
+ && nextRetryDelay == that.nextRetryDelay
+ && Objects.equals(sessionId, that.sessionId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(sequence, compressFlag, sentTime, retryCount, firstSendTime, nextRetryDelay, sessionId);
+ }
+
+ @Override
+ public String toString() {
+ return "PendingMessage{" + "sequence="
+ + sequence + ", messageLength="
+ + message.length + ", compressFlag="
+ + compressFlag + ", sentTime="
+ + sentTime + ", retryCount="
+ + retryCount + ", firstSendTime="
+ + firstSendTime + ", nextRetryDelay="
+ + nextRetryDelay + ", sessionId='"
+ + sessionId + '\'' + '}';
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/PendingMessageStore.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/PendingMessageStore.java
new file mode 100644
index 0000000000..7b5e92c82b
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/PendingMessageStore.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.store;
+
+import org.apache.dubbo.common.extension.ExtensionScope;
+import org.apache.dubbo.common.extension.SPI;
+
+/**
+ * SPI interface for pending message store in reliable streaming.
+ * Provides persistence and recovery capabilities for unacknowledged messages.
+ */
+@SPI(scope = ExtensionScope.FRAMEWORK)
+public interface PendingMessageStore {
+
+ /**
+ * Initialize the store for a session
+ *
+ * @param metadata session metadata
+ * @throws StoreException if initialization fails
+ */
+ void init(SessionMetadata metadata) throws StoreException;
+
+ /**
+ * Store a pending message
+ *
+ * @param message the pending message to store
+ * @throws StoreException if storage fails
+ */
+ void put(PendingMessage message) throws StoreException;
+
+ /**
+ * Acknowledge a message by sequence number
+ *
+ * @param sequence the sequence number to acknowledge
+ * @throws StoreException if ack operation fails
+ */
+ void ack(long sequence) throws StoreException;
+
+ /**
+ * Remove a message by sequence number
+ *
+ * @param sequence the sequence number to remove
+ * @throws StoreException if removal fails
+ */
+ void remove(long sequence) throws StoreException;
+
+ /**
+ * Load all pending messages for a session
+ *
+ * @param sessionId the session id
+ * @return list of pending messages, empty list if none found
+ * @throws StoreException if loading fails
+ */
+ java.util.List<PendingMessage> load(String sessionId) throws StoreException;
+
+ /**
+ * Close the store for a session
+ *
+ * @param metadata session metadata
+ * @throws StoreException if close fails
+ */
+ void close(SessionMetadata metadata) throws StoreException;
+
+ /**
+ * Check if the store is healthy
+ *
+ * @return true if healthy, false otherwise
+ */
+ boolean isHealthy();
+
+ /**
+ * Load session metadata for recovery after process restart.
+ * Returns null if no metadata found for the session.
+ *
+ * @param sessionId the session id
+ * @return session metadata, or null if not found
+ * @throws StoreException if loading fails
+ */
+ SessionMetadata loadMetadata(String sessionId) throws StoreException;
+
+ /**
+ * Update session metadata (lastAckedSeq, currentSeq, etc.)
+ * This is called periodically to persist critical state for process restart recovery.
+ *
+ * @param metadata the updated session metadata
+ * @throws StoreException if update fails
+ */
+ void updateMetadata(SessionMetadata metadata) throws StoreException;
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/SessionMetadata.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/SessionMetadata.java
new file mode 100644
index 0000000000..e374e0344d
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/SessionMetadata.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.store;
+
+import org.apache.dubbo.common.URL;
+import org.apache.dubbo.rpc.protocol.tri.stream.ReliabilityConfig;
+
+import java.util.Objects;
+
+/**
+ * Metadata for a reliable streaming session.
+ * Includes critical state information for process restart recovery.
+ */
+public class SessionMetadata {
+
+ private final String sessionId;
+ private final String serviceKey;
+ private final long createTime;
+ private final URL url;
+ private final ReliabilityConfig config;
+
+ // Critical state for recovery after process restart
+ private final long lastAckedSeq; // Last acknowledged sequence number
+ private final long currentSeq; // Current sequence number
+ private final long lastUpdateTime; // Last update timestamp
+
+ /**
+ * Constructor for new session (no recovery)
+ */
+ public SessionMetadata(String sessionId, String serviceKey, long createTime, URL url, ReliabilityConfig config) {
+ this(sessionId, serviceKey, createTime, url, config, 0L, 0L, System.currentTimeMillis());
+ }
+
+ /**
+ * Constructor for session with state (for recovery)
+ */
+ public SessionMetadata(
+ String sessionId,
+ String serviceKey,
+ long createTime,
+ URL url,
+ ReliabilityConfig config,
+ long lastAckedSeq,
+ long currentSeq,
+ long lastUpdateTime) {
+ this.sessionId = Objects.requireNonNull(sessionId, "sessionId cannot be null");
+ this.serviceKey = Objects.requireNonNull(serviceKey, "serviceKey cannot be null");
+ this.createTime = createTime;
+ this.url = Objects.requireNonNull(url, "url cannot be null");
+ this.config = Objects.requireNonNull(config, "config cannot be null");
+ this.lastAckedSeq = lastAckedSeq;
+ this.currentSeq = currentSeq;
+ this.lastUpdateTime = lastUpdateTime;
+ }
+
+ public String getSessionId() {
+ return sessionId;
+ }
+
+ public String getServiceKey() {
+ return serviceKey;
+ }
+
+ public long getCreateTime() {
+ return createTime;
+ }
+
+ public URL getUrl() {
+ return url;
+ }
+
+ public ReliabilityConfig getConfig() {
+ return config;
+ }
+
+ public long getLastAckedSeq() {
+ return lastAckedSeq;
+ }
+
+ public long getCurrentSeq() {
+ return currentSeq;
+ }
+
+ public long getLastUpdateTime() {
+ return lastUpdateTime;
+ }
+
+ /**
+ * Create a new SessionMetadata with updated state
+ */
+ public SessionMetadata withUpdatedState(long newLastAckedSeq, long newCurrentSeq) {
+ return new SessionMetadata(
+ sessionId,
+ serviceKey,
+ createTime,
+ url,
+ config,
+ newLastAckedSeq,
+ newCurrentSeq,
+ System.currentTimeMillis());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ SessionMetadata that = (SessionMetadata) o;
+ return createTime == that.createTime
+ && lastAckedSeq == that.lastAckedSeq
+ && currentSeq == that.currentSeq
+ && lastUpdateTime == that.lastUpdateTime
+ && Objects.equals(sessionId, that.sessionId)
+ && Objects.equals(serviceKey, that.serviceKey)
+ && Objects.equals(url, that.url)
+ && Objects.equals(config, that.config);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(sessionId, serviceKey, createTime, url, config, lastAckedSeq, currentSeq, lastUpdateTime);
+ }
+
+ @Override
+ public String toString() {
+ return "SessionMetadata{" + "sessionId='"
+ + sessionId + '\'' + ", serviceKey='"
+ + serviceKey + '\'' + ", createTime="
+ + createTime + ", lastAckedSeq="
+ + lastAckedSeq + ", currentSeq="
+ + currentSeq + ", lastUpdateTime="
+ + lastUpdateTime + ", url="
+ + url + ", config="
+ + config + '}';
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/StoreException.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/StoreException.java
new file mode 100644
index 0000000000..a1d92b8b87
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/store/StoreException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.store;
+
+/**
+ * Exception thrown by PendingMessageStore operations
+ */
+public class StoreException extends Exception {
+
+ public StoreException(String message) {
+ super(message);
+ }
+
+ public StoreException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public StoreException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/AbstractTripleClientStream.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/AbstractTripleClientStream.java
index 8b677e74ee..33dc249c80 100644
--- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/AbstractTripleClientStream.java
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/AbstractTripleClientStream.java
@@ -16,7 +16,9 @@
*/
package org.apache.dubbo.rpc.protocol.tri.stream;
+import org.apache.dubbo.common.URL;
import org.apache.dubbo.common.constants.CommonConstants;
+import org.apache.dubbo.common.extension.ExtensionLoader;
import org.apache.dubbo.common.logger.ErrorTypeAwareLogger;
import org.apache.dubbo.common.logger.LoggerFactory;
import org.apache.dubbo.remoting.Constants;
@@ -25,6 +27,7 @@ import org.apache.dubbo.rpc.TriRpcStatus;
import org.apache.dubbo.rpc.model.FrameworkModel;
import org.apache.dubbo.rpc.protocol.tri.ClassLoadUtil;
import org.apache.dubbo.rpc.protocol.tri.ExceptionUtils;
+import org.apache.dubbo.rpc.protocol.tri.RequestMetadata;
import org.apache.dubbo.rpc.protocol.tri.TripleHeaderEnum;
import org.apache.dubbo.rpc.protocol.tri.command.CancelQueueCommand;
import org.apache.dubbo.rpc.protocol.tri.command.DataQueueCommand;
@@ -35,6 +38,9 @@ import org.apache.dubbo.rpc.protocol.tri.compressor.Identity;
import org.apache.dubbo.rpc.protocol.tri.frame.Deframer;
import org.apache.dubbo.rpc.protocol.tri.frame.TriDecoder;
import org.apache.dubbo.rpc.protocol.tri.h12.grpc.GrpcUtils;
+import org.apache.dubbo.rpc.protocol.tri.store.PendingMessageStore;
+import org.apache.dubbo.rpc.protocol.tri.store.SessionMetadata;
+import org.apache.dubbo.rpc.protocol.tri.store.StoreException;
import org.apache.dubbo.rpc.protocol.tri.transport.AbstractH2TransportListener;
import org.apache.dubbo.rpc.protocol.tri.transport.H2TransportListener;
import org.apache.dubbo.rpc.protocol.tri.transport.TripleWriteQueue;
@@ -47,7 +53,18 @@ import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
import com.google.protobuf.Any;
import com.google.rpc.DebugInfo;
@@ -62,6 +79,7 @@ import io.netty.handler.codec.http2.Http2StreamChannel;
import io.netty.util.AttributeKey;
import io.netty.util.ReferenceCountUtil;
+import static org.apache.dubbo.common.constants.LoggerCodeConstants.INTERNAL_ERROR;
import static org.apache.dubbo.common.constants.LoggerCodeConstants.PROTOCOL_FAILED_RESPONSE;
/**
@@ -69,22 +87,70 @@ import static org.apache.dubbo.common.constants.LoggerCodeConstants.PROTOCOL_FAI
* write Http2Frame to remote. A {@link H2TransportListener} receives Http2Frame from remote.
* Instead of maintaining state, this class depends on upper layer or transport layer's states.
*/
-public abstract class AbstractTripleClientStream extends AbstractStream implements ClientStream {
+public abstract class AbstractTripleClientStream extends AbstractStream implements ClientStream, ReliabilityContext {
private static final ErrorTypeAwareLogger LOGGER =
LoggerFactory.getErrorTypeAwareLogger(AbstractTripleClientStream.class);
private static final AttributeKey<SSLSession> SSL_SESSION_KEY = AttributeKey.valueOf(Constants.SSL_SESSION_KEY);
private final ClientStream.Listener listener;
- protected final TripleWriteQueue writeQueue;
+ protected volatile TripleWriteQueue writeQueue;
private Deframer deframer;
- private final Channel parent;
- private final TripleStreamChannelFuture streamChannelFuture;
+ private volatile Channel parent;
+ private final AtomicReference<TripleStreamChannelFuture> streamChannelFuture = new AtomicReference<>();
private boolean halfClosed;
private boolean rst;
private boolean isReturnTriException = false;
+ // Reliability fields (lazy initialization)
+ private volatile boolean reliabilityEnabled = false;
+ private AtomicLong sequenceNumber;
+ private ConcurrentHashMap<Long, PendingMessage> pendingMessages;
+ private AtomicLong ackWatermark; // 水位标记,用于O(1)清理
+ private PendingMessageStore messageStore; // 持久化存储
+ private SessionMetadata currentMetadata; // Current session metadata for persistence
+
+ // ReliabilityContext support fields
+ private final AtomicLong totalSentCount = new AtomicLong(0);
+ private final AtomicLong totalRetryCount = new AtomicLong(0);
+ private volatile Consumer<String> stateChangeCallback;
+ private volatile Consumer<String> recoveryCallback;
+ private volatile Consumer<Long> retryCallback;
+ private String sessionId;
+ private ReliabilityConfig config;
+ private URL url; // Store URL for message store initialization
+ private ScheduledExecutorService heartbeatScheduler; // 心跳专用调度器
+ private ScheduledExecutorService retryScheduler; // 重试检查专用调度器
+ private ScheduledExecutorService recoveryExecutor; // 恢复操作专用调度器(支持延迟恢复)
+
+ // Track scheduled tasks for proper cancellation
+ private volatile ScheduledFuture<?> currentHeartbeatTask;
+ private volatile ScheduledFuture<?> currentRetryTask;
+ private final Set<ScheduledFuture<?>> activeTasks = ConcurrentHashMap.newKeySet();
+ private ReconnectionManager reconnectionManager; // 重连管理器
+ private volatile long lastHeartbeatAckTime = System.currentTimeMillis();
+ private final AtomicLong lastAckedSeq = new AtomicLong(0);
+ private final AtomicInteger inFlightCount = new AtomicInteger(0); // InFlight 消息计数器
+ private int maxRetryTimes; // 最大重试次数
+
+ // Transport listener for handling incoming data
+ private H2TransportListener transportListener;
+
+ // Heartbeat state machine
+ private volatile HeartbeatState heartbeatState = HeartbeatState.HEALTHY;
+ private volatile int missedHeartbeats = 0;
+ private volatile long lastHeartbeatSentTime = 0;
+
+ enum HeartbeatState {
+ HEALTHY, // Normal operation
+ SUSPECT, // Missed heartbeats, monitoring closely
+ RECONNECTING, // Attempting to reconnect
+ FAILED, // Connection failed, stop trying
+ PAUSED, // Temporarily paused but can be recovered
+ CLOSED // Stream closed, no further operations
+ }
+
protected AbstractTripleClientStream(
FrameworkModel frameworkModel,
Executor executor,
@@ -95,7 +161,8 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
this.parent = http2StreamChannel.parent();
this.listener = listener;
this.writeQueue = writeQueue;
- this.streamChannelFuture = initStreamChannel(http2StreamChannel);
+ this.streamChannelFuture.set(initStreamChannel(http2StreamChannel));
+ this.transportListener = createTransportListener();
}
protected AbstractTripleClientStream(
@@ -108,7 +175,343 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
this.parent = parent;
this.listener = listener;
this.writeQueue = writeQueue;
- this.streamChannelFuture = initStreamChannel(parent);
+ this.streamChannelFuture.set(initStreamChannel(parent));
+ this.transportListener = createTransportListener();
+ }
+
+ /**
+ * Unified reliability initialization method to prevent duplicate initialization.
+ * This method contains all the core initialization logic including persistence, recovery, and scheduling.
+ */
+ private void doInitializeReliability(String sessionId, ReliabilityConfig config, URL url) {
+ // Use atomic flag to prevent duplicate initialization
+ if (!reliabilityInitialized.compareAndSet(false, true)) {
+ LOGGER.debug(
+ "Reliability already initialized for session: {}, skipping duplicate initialization", sessionId);
+ return;
+ }
+
+ try {
+ LOGGER.info("Starting unified reliability initialization for session: {}", sessionId);
+
+ // Set core reliability state
+ this.reliabilityEnabled = true;
+ this.sessionId = sessionId;
+ this.url = url;
+ this.config = config;
+
+ // Initialize with default values (will be overridden if recovery data exists)
+ this.sequenceNumber = new AtomicLong(0);
+ this.pendingMessages = new ConcurrentHashMap<>();
+ this.ackWatermark = new AtomicLong(0);
+ this.lastHeartbeatAckTime = System.currentTimeMillis();
+ this.missedHeartbeats = 0;
+ this.heartbeatState = HeartbeatState.HEALTHY;
+
+ // Initialize thread pools and schedulers
+ initializeExecutors();
+
+ // Initialize persistent message store
+ initializeMessageStore();
+
+ // Try to recover session state from persistent storage (for process restart)
+ recoverSessionStateFromStorage();
+
+ // Recover any pending messages from storage
+ recoverPendingMessages();
+
+ // Start heartbeat and retry scheduling
+ scheduleHeartbeat();
+ scheduleRetryCheck();
+
+ LOGGER.info("Successfully completed unified reliability initialization for session: {}", sessionId);
+
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Failed to initialize reliability for session: " + sessionId, e);
+ // Reset the flag to allow retry
+ reliabilityInitialized.set(false);
+ // Clean up any partial initialization
+ cleanupReliabilityResources();
+ throw new RuntimeException("Reliability initialization failed", e);
+ }
+ }
+
+ /**
+ * Initialize executors and schedulers for reliability features.
+ */
+ private void initializeExecutors() {
+ // Use safe session identifier to handle potential null sessionId
+ String safeSessionId = (sessionId != null) ? sessionId : "temp-" + System.nanoTime();
+
+ this.heartbeatScheduler = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-heartbeat-" + safeSessionId);
+ t.setDaemon(true);
+ return t;
+ });
+
+ this.retryScheduler = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-retry-" + safeSessionId);
+ t.setDaemon(true);
+ return t;
+ });
+
+ this.recoveryExecutor = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-recovery-" + safeSessionId);
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ public void initializeReliability(RequestMetadata requestMetadata, URL url) {
+ Map<String, Object> attachments = requestMetadata.attachments;
+ boolean hasReliabilityInMetadata = attachments != null && attachments.containsKey("tri-reliable-version");
+ boolean hasSessionIdFromHeaders = this.sessionId != null && !this.sessionId.startsWith("metadata-session-");
+
+ if (hasReliabilityInMetadata || hasSessionIdFromHeaders) {
+ // Extract session ID from metadata or use existing header-based session ID
+ String sessionId;
+ if (hasReliabilityInMetadata) {
+ sessionId = (String) attachments.get("tri-session-id");
+ if (sessionId == null) {
+ sessionId = hasSessionIdFromHeaders ? this.sessionId : "metadata-session-" + System.nanoTime();
+ LOGGER.warn(INTERNAL_ERROR, "", "", "No session ID in request metadata, using: " + sessionId);
+ }
+ } else {
+ sessionId = this.sessionId; // Use session ID from headers
+ LOGGER.info("Using session ID from headers for deferred reliability initialization: {}", sessionId);
+ }
+
+ ReliabilityConfig config = parseReliabilityConfig(url);
+
+ // Call unified initialization method with proper URL
+ doInitializeReliability(sessionId, config, url);
+
+ LOGGER.info("Reliability initialized from RequestMetadata for session: {}", sessionId);
+ }
+ }
+
+ private void initializeReliabilityFromHeaders(Http2Headers headers) {
+ // Check if headers contain reliability negotiation information
+ if (headers != null && headers.contains("tri-reliable-version")) {
+ // Extract session ID from headers
+ String headerSessionId = headers.get("tri-session-id") != null
+ ? headers.get("tri-session-id").toString()
+ : "header-session-" + System.nanoTime();
+
+ // If reliability is not yet initialized, perform full initialization
+ if (!reliabilityInitialized.get()) {
+ // Only initialize from headers if we have a real URL context
+ if (this.url != null) {
+ LOGGER.info(
+ "Initializing reliability from headers for session: {} with URL: {}",
+ headerSessionId,
+ this.url);
+
+ ReliabilityConfig config = parseReliabilityConfig(this.url);
+
+ // Call unified initialization method
+ doInitializeReliability(headerSessionId, config, this.url);
+
+ LOGGER.info("Reliability initialized from headers for session: {}", headerSessionId);
+ } else {
+ // Defer initialization until proper URL is available
+ LOGGER.info(
+ "Deferring reliability initialization from headers for session: {} - waiting for URL context",
+ headerSessionId);
+
+ // Just store the session ID for later use
+ this.sessionId = headerSessionId;
+ // Mark that we have seen reliability headers but haven't initialized yet
+ // This will be picked up by initializeReliability() when it's called with proper URL
+ }
+ } else {
+ // If already initialized, just update session ID if it was not set
+ if (this.sessionId == null || this.sessionId.startsWith("unknown-session")) {
+ this.sessionId = headerSessionId;
+ LOGGER.info("Updated session ID from headers: {}", headerSessionId);
+ } else {
+ LOGGER.debug(
+ "Reliability already initialized for session: {}, preserving existing configuration",
+ this.sessionId);
+ }
+ }
+ }
+ }
+
+ /**
+ * Initialize the message store based on configuration
+ */
+ private void initializeMessageStore() {
+ try {
+ String storeType = config.getStoreType();
+ ExtensionLoader<PendingMessageStore> loader = ExtensionLoader.getExtensionLoader(PendingMessageStore.class);
+ this.messageStore = loader.getExtension(storeType);
+
+ SessionMetadata metadata = new SessionMetadata(
+ sessionId, url != null ? url.getServiceKey() : "unknown", System.currentTimeMillis(), url, config);
+ this.currentMetadata = metadata; // Store reference for later updates
+ messageStore.init(metadata);
+
+ LOGGER.info("Initialized message store of type: {} for session: {}", storeType, sessionId);
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Failed to initialize message store for session: " + sessionId, e);
+ // Fallback to in-memory store
+ this.messageStore = new org.apache.dubbo.rpc.protocol.tri.store.InMemoryPendingMessageStore();
+ try {
+ SessionMetadata metadata = new SessionMetadata(
+ sessionId,
+ url != null ? url.getServiceKey() : "unknown",
+ System.currentTimeMillis(),
+ url,
+ config);
+ this.currentMetadata = metadata; // Store reference for later updates
+ messageStore.init(metadata);
+ } catch (Exception ex) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to initialize fallback in-memory store for session: " + sessionId,
+ ex);
+ }
+ }
+ }
+
+ /**
+ * Recover session state (lastAckedSeq, currentSeq) from persistent storage.
+ * This is critical for process restart recovery to avoid duplicate sends and maintain sequence continuity.
+ */
+ private void recoverSessionStateFromStorage() {
+ try {
+ if (messageStore == null) {
+ LOGGER.debug("No message store available for session state recovery: {}", sessionId);
+ return;
+ }
+
+ // Try to load persisted metadata
+ org.apache.dubbo.rpc.protocol.tri.store.SessionMetadata recoveredMetadata =
+ messageStore.loadMetadata(sessionId);
+
+ if (recoveredMetadata == null) {
+ LOGGER.info("No persisted session state found for session: {}, starting fresh", sessionId);
+ return;
+ }
+
+ // Recover lastAckedSeq
+ long recoveredLastAcked = recoveredMetadata.getLastAckedSeq();
+ if (recoveredLastAcked > 0) {
+ this.lastAckedSeq.set(recoveredLastAcked);
+ LOGGER.info(
+ "Recovered lastAckedSeq = {} for session: {} from persistent storage",
+ recoveredLastAcked,
+ sessionId);
+ }
+
+ // Recover sequenceNumber (currentSeq)
+ long recoveredCurrentSeq = recoveredMetadata.getCurrentSeq();
+ if (recoveredCurrentSeq > 0) {
+ this.sequenceNumber.set(recoveredCurrentSeq);
+ LOGGER.info(
+ "Recovered currentSeq = {} for session: {} from persistent storage",
+ recoveredCurrentSeq,
+ sessionId);
+ }
+
+ LOGGER.info(
+ "Successfully recovered session state for session: {} (lastAcked={}, currentSeq={})",
+ sessionId,
+ recoveredLastAcked,
+ recoveredCurrentSeq);
+
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to recover session state for session: " + sessionId + ", continuing with fresh state",
+ e);
+ // Continue with fresh state - don't fail initialization
+ }
+ }
+
+ /**
+ * Recover pending messages from persistent storage
+ */
+ private void recoverPendingMessages() {
+ try {
+ if (messageStore == null) {
+ LOGGER.debug("No message store available for session: {}", sessionId);
+ return;
+ }
+
+ java.util.List<org.apache.dubbo.rpc.protocol.tri.store.PendingMessage> recoveredMessages =
+ messageStore.load(sessionId);
+
+ if (recoveredMessages.isEmpty()) {
+ LOGGER.debug("No messages to recover for session: {}", sessionId);
+ return;
+ }
+
+ // 合并恢复的消息到内存pendingMessages
+ for (org.apache.dubbo.rpc.protocol.tri.store.PendingMessage recovered : recoveredMessages) {
+ // 以序列号最大的为准,避免重复
+ pendingMessages.merge(
+ recovered.getSequence(), convertToInternalPendingMessage(recovered), (existing, newMsg) -> {
+ // 保留sentTime较新的那个
+ return existing.getSentTime() > newMsg.getSentTime() ? existing : newMsg;
+ });
+ }
+
+ LOGGER.info("Recovered {} messages for session: {}", recoveredMessages.size(), sessionId);
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Failed to recover pending messages for session: " + sessionId, e);
+ }
+ }
+
+ /**
+ * Convert store PendingMessage to internal PendingMessage
+ */
+ private PendingMessage convertToInternalPendingMessage(
+ org.apache.dubbo.rpc.protocol.tri.store.PendingMessage storeMessage) {
+ return new PendingMessage(
+ storeMessage.getSequence(),
+ storeMessage.getMessage(),
+ storeMessage.getCompressFlag(),
+ storeMessage.getSentTime(),
+ storeMessage.getRetryCount());
+ }
+
+ /**
+ * Set the reconnection manager for handling connection failures.
+ * This allows the stream to attempt real reconnection when the underlying connection fails.
+ *
+ * @param reconnectionManager the reconnection manager
+ */
+ public void setReconnectionManager(ReconnectionManager reconnectionManager) {
+ this.reconnectionManager = reconnectionManager;
+ }
+
+ private ReliabilityConfig parseReliabilityConfig(URL url) {
+ return ReliabilityConfig.builder()
+ .enabled(url.getParameter("stream.reliability.enabled", false))
+ .heartbeatInterval(url.getParameter("stream.heartbeat.interval", 5000))
+ .retryTimeout(url.getParameter("stream.retry.timeout", 10000))
+ .maxRetries(url.getParameter("stream.max.retries", 3))
+ .sessionTimeout(url.getParameter("stream.session.timeout", 30000))
+ .maxMissedHeartbeats(url.getParameter("stream.heartbeat.max-missed", 3))
+ // 智能重试策略参数 - 开放给用户配置
+ .initialRetryDelay(url.getParameter("stream.retry.initial-delay", 1000))
+ .maxRetryDelay(url.getParameter("stream.retry.max-delay", 30000))
+ .backoffMultiplier(url.getParameter("stream.retry.backoff-multiplier", 2.0))
+ .maxInFlightMessages(url.getParameter("stream.max-in-flight", 100))
+ .totalRetryTimeout(url.getParameter("stream.retry.total-timeout", 60000))
+ // 持久化存储配置 - 开放给用户配置
+ .storeType(url.getParameter("stream.reliability.store.type", "memory"))
+ .storePath(url.getParameter("stream.reliability.store.path", "/tmp/dubbo/reliable"))
+ .storeMaxFileSize(url.getParameter("stream.reliability.store.max-file-size", 100 * 1024 * 1024L))
+ .storeRetentionTime(url.getParameter("stream.reliability.store.retention-time", 24 * 60 * 60 * 1000L))
+ .storeSyncInterval(url.getParameter("stream.reliability.store.sync-interval", 1000))
+ .build();
}
protected abstract TripleStreamChannelFuture initStreamChannel(Channel parent);
@@ -122,7 +525,13 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
if (!checkResult.isSuccess()) {
return checkResult;
}
- final HeaderQueueCommand headerCmd = HeaderQueueCommand.createHeaders(streamChannelFuture, headers);
+
+ // Initialize reliability features from headers
+ if (headers != null && headers.contains("tri-reliable-version")) {
+ initializeReliabilityFromHeaders(headers);
+ }
+
+ final HeaderQueueCommand headerCmd = HeaderQueueCommand.createHeaders(getCurrentStreamChannelFuture(), headers);
return writeQueue.enqueueFuture(headerCmd, parent.eventLoop()).addListener(future -> {
if (!future.isSuccess()) {
transportException(future.cause());
@@ -133,7 +542,31 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
private void transportException(Throwable cause) {
final TriRpcStatus status =
TriRpcStatus.INTERNAL.withDescription("Http2 exception").withCause(cause);
- listener.onComplete(status, null, null, false);
+
+ // Classify error: recoverable (network issue) vs non-recoverable (protocol error)
+ if (isRecoverableException(cause)) {
+ // Recoverable: Handle as temporary failure, trigger recovery
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Recoverable transport exception for session: " + sessionId + " - " + cause.getMessage());
+ handleTemporaryFailureWithCount("Transport exception: " + cause.getMessage());
+
+ // Do NOT call onComplete - let recovery mechanism handle it
+ // The stream remains alive for retry/reconnect
+ } else {
+ // Non-recoverable: Terminate the stream immediately
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Non-recoverable transport exception for session: " + sessionId + " - " + cause.getMessage());
+
+ // Clean up resources and notify business layer
+ cleanupReliabilityResources();
+ listener.onComplete(status, null, null, false);
+ }
}
public ChannelFuture cancelByLocal(TriRpcStatus status) {
@@ -141,7 +574,21 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
if (!checkResult.isSuccess()) {
return checkResult;
}
- final CancelQueueCommand cmd = CancelQueueCommand.createCommand(streamChannelFuture, Http2Error.CANCEL);
+
+ // Clean up reliability resources
+ if (reliabilityEnabled || reliabilityInitialized.get()) {
+ cleanupReliabilityResources();
+ }
+
+ // Handle case where stream channel future is null (e.g., after connection failure)
+ TripleStreamChannelFuture currentFuture = getCurrentStreamChannelFuture();
+ if (currentFuture == null) {
+ LOGGER.debug("Cannot cancel stream - no active stream channel future for session: {}", sessionId);
+ rst = true;
+ return checkResult; // Return the successful preCheck result
+ }
+
+ final CancelQueueCommand cmd = CancelQueueCommand.createCommand(currentFuture, Http2Error.CANCEL);
rst = true;
return this.writeQueue.enqueue(cmd);
@@ -152,61 +599,2282 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
return parent.remoteAddress();
}
- @Override
- public SSLSession getSslSession() {
- return parent.attr(SSL_SESSION_KEY).get();
+ @Override
+ public SSLSession getSslSession() {
+ return parent.attr(SSL_SESSION_KEY).get();
+ }
+
+ @Override
+ public ChannelFuture sendMessage(byte[] message, int compressFlag) {
+ ChannelFuture checkResult = preCheck();
+ if (!checkResult.isSuccess()) {
+ return checkResult;
+ }
+
+ if (!reliabilityEnabled) {
+ // Original path for non-reliable mode
+ final DataQueueCommand cmd =
+ DataQueueCommand.create(getCurrentStreamChannelFuture(), message, false, compressFlag);
+ return this.writeQueue.enqueueFuture(cmd, parent.eventLoop()).addListener(future -> {
+ if (!future.isSuccess()) {
+ cancelByLocal(TriRpcStatus.INTERNAL
+ .withDescription("Client write message failed")
+ .withCause(future.cause()));
+ transportException(future.cause());
+ }
+ });
+ }
+
+ // Reliable mode: send sequence header first, then data
+
+ // Check InFlight limit before sending
+ int currentInFlight = inFlightCount.get();
+ if (currentInFlight >= config.getMaxInFlightMessages()) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "InFlight limit exceeded: " + currentInFlight + "/" + config.getMaxInFlightMessages()
+ + " messages, rejecting new message");
+ io.netty.channel.ChannelPromise rejectPromise = parent.newPromise();
+ rejectPromise.setFailure(new IllegalStateException("InFlight message limit exceeded"));
+ return rejectPromise;
+ }
+
+ long seq = sequenceNumber.incrementAndGet();
+
+ // Create sequence header
+ io.netty.handler.codec.http2.DefaultHttp2Headers seqHeaders =
+ new io.netty.handler.codec.http2.DefaultHttp2Headers();
+ seqHeaders.set("tri-seq", String.valueOf(seq));
+ HeaderQueueCommand headerCmd =
+ HeaderQueueCommand.createHeaders(getCurrentStreamChannelFuture(), seqHeaders, false);
+
+ // Cache pending message
+ PendingMessage pending = new PendingMessage(seq, message, compressFlag, System.currentTimeMillis(), 0);
+
+ // 持久化存储消息
+ if (messageStore != null && messageStore.isHealthy()) {
+ try {
+ org.apache.dubbo.rpc.protocol.tri.store.PendingMessage storeMessage =
+ new org.apache.dubbo.rpc.protocol.tri.store.PendingMessage(
+ seq,
+ message,
+ compressFlag,
+ System.currentTimeMillis(),
+ 0,
+ System.currentTimeMillis(),
+ 100,
+ sessionId);
+ messageStore.put(storeMessage);
+ } catch (StoreException e) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to persist message with sequence: " + seq + " for session: " + sessionId,
+ e);
+ }
+ }
+
+ pendingMessages.put(seq, pending);
+
+ // Create combined promise for header + data
+ io.netty.channel.ChannelPromise combinedPromise = parent.newPromise();
+ ChannelFuture headerFuture = writeQueue.enqueueFuture(headerCmd, parent.eventLoop());
+
+ headerFuture.addListener(future -> {
+ if (future.isSuccess()) {
+ DataQueueCommand dataCmd =
+ DataQueueCommand.create(getCurrentStreamChannelFuture(), message, false, compressFlag);
+ ChannelFuture dataFuture = writeQueue.enqueueFuture(dataCmd, parent.eventLoop());
+ dataFuture.addListener(dataResult -> {
+ if (dataResult.isSuccess()) {
+ // Message successfully sent to network, increment InFlight count
+ inFlightCount.incrementAndGet();
+ incrementTotalSentCount();
+ combinedPromise.setSuccess();
+ } else {
+ combinedPromise.setFailure(dataResult.cause());
+ pendingMessages.remove(seq);
+ // Decrement InFlight count since message failed to send
+ inFlightCount.decrementAndGet();
+ cancelByLocal(TriRpcStatus.INTERNAL
+ .withDescription("Client write data failed")
+ .withCause(dataResult.cause()));
+ }
+ });
+ } else {
+ combinedPromise.setFailure(future.cause());
+ pendingMessages.remove(seq);
+ // Note: InFlight count not incremented here since data send failed before increment
+ cancelByLocal(TriRpcStatus.INTERNAL
+ .withDescription("Client write header failed")
+ .withCause(future.cause()));
+ }
+ });
+
+ return combinedPromise;
+ }
+
+ @Override
+ public void request(int n) {
+ deframer.request(n);
+ }
+
+ @Override
+ public ChannelFuture halfClose() {
+ ChannelFuture checkResult = preCheck();
+ if (!checkResult.isSuccess()) {
+ return checkResult;
+ }
+ final EndStreamQueueCommand cmd = EndStreamQueueCommand.create(getCurrentStreamChannelFuture());
+ return this.writeQueue.enqueueFuture(cmd, parent.eventLoop()).addListener(future -> {
+ if (future.isSuccess()) {
+ halfClosed = true;
+ // Clean up reliability resources when half-closing
+ cleanupReliabilityResources();
+ }
+ });
+ }
+
+ private ChannelFuture preCheck() {
+ if (rst) {
+ TripleStreamChannelFuture currentFuture = getCurrentStreamChannelFuture();
+ if (currentFuture != null) {
+ Channel channel = currentFuture.getNow();
+ if (channel != null) {
+ return channel.newFailedFuture(new IOException("stream channel has reset"));
+ } else {
+ // If channel is not ready yet but stream is reset, use parent channel
+ return parent.newFailedFuture(new IOException("stream channel has reset"));
+ }
+ }
+ }
+ return parent.newSucceededFuture();
+ }
+
+ private void scheduleHeartbeat() {
+ if (!reliabilityEnabled
+ || heartbeatScheduler == null
+ || heartbeatScheduler.isShutdown()
+ || heartbeatState == HeartbeatState.CLOSED) {
+ return;
+ }
+
+ try {
+ // Cancel existing heartbeat task if any
+ if (currentHeartbeatTask != null && !currentHeartbeatTask.isDone()) {
+ currentHeartbeatTask.cancel(false);
+ LOGGER.debug("Cancelled previous heartbeat task for session: {}", sessionId);
+ }
+
+ // Schedule new heartbeat task with tracking
+ currentHeartbeatTask = scheduleTrackedTaskAtFixedRate(
+ heartbeatScheduler,
+ () -> {
+ try {
+ checkHeartbeatState();
+ sendHeartbeatIfNeeded();
+ } catch (Exception e) {
+ LOGGER.warn(
+ INTERNAL_ERROR, "", "", "Heartbeat monitoring failed for session: " + sessionId, e);
+ }
+ },
+ config.getHeartbeatInterval(),
+ config.getHeartbeatInterval(),
+ TimeUnit.MILLISECONDS);
+
+ if (currentHeartbeatTask != null) {
+ LOGGER.debug("Scheduled new heartbeat task for session: {}", sessionId);
+ }
+ } catch (java.util.concurrent.RejectedExecutionException e) {
+ LOGGER.debug("Failed to schedule heartbeat - scheduler shut down for session: {}", sessionId);
+ }
+ }
+
+ private void checkHeartbeatState() {
+ long now = System.currentTimeMillis();
+ long timeSinceLastAck = now - lastHeartbeatAckTime;
+
+ switch (heartbeatState) {
+ case HEALTHY:
+ if (timeSinceLastAck > config.getHeartbeatInterval() * 2) {
+ missedHeartbeats++;
+ if (missedHeartbeats >= 1) {
+ transitionToState(HeartbeatState.SUSPECT, "First heartbeat timeout");
+ }
+ }
+ break;
+
+ case SUSPECT:
+ if (timeSinceLastAck > config.getSessionTimeout() / 2) {
+ missedHeartbeats++;
+ if (missedHeartbeats >= config.getMaxMissedHeartbeats()) {
+ transitionToState(HeartbeatState.RECONNECTING, "Max missed heartbeats exceeded");
+ }
+ }
+ break;
+
+ case RECONNECTING:
+ if (timeSinceLastAck > config.getSessionTimeout()) {
+ transitionToState(HeartbeatState.FAILED, "Session timeout exceeded");
+ }
+ break;
+
+ case FAILED:
+ // Stay in failed state
+ break;
+ }
+ }
+
+ private void sendHeartbeatIfNeeded() {
+ if (heartbeatState == HeartbeatState.FAILED || !parent.isActive()) {
+ return;
+ }
+
+ long now = System.currentTimeMillis();
+ if (now - lastHeartbeatSentTime >= config.getHeartbeatInterval()) {
+ io.netty.handler.codec.http2.DefaultHttp2Headers heartbeat =
+ new io.netty.handler.codec.http2.DefaultHttp2Headers();
+ heartbeat.set("tri-heartbeat", String.valueOf(now));
+ heartbeat.set("tri-last-acked", String.valueOf(getLastAckedSeq()));
+ heartbeat.set("tri-state", heartbeatState.name().toLowerCase());
+
+ HeaderQueueCommand cmd =
+ HeaderQueueCommand.createHeaders(getCurrentStreamChannelFuture(), heartbeat, false);
+ writeQueue.enqueueFuture(cmd, parent.eventLoop()).addListener(future -> {
+ if (!future.isSuccess()) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Failed to send heartbeat", future.cause());
+ }
+ });
+ lastHeartbeatSentTime = now;
+ }
+ }
+
+ /**
+ * Send a heartbeat with custom headers.
+ * This is used for recovery negotiation where additional context is needed.
+ */
+ private void sendHeartbeatWithHeaders(java.util.Map<CharSequence, Object> customHeaders) {
+ if (heartbeatState == HeartbeatState.FAILED || !parent.isActive()) {
+ LOGGER.debug("Skipping heartbeat with headers - connection not active or failed state");
+ return;
+ }
+
+ try {
+ long now = System.currentTimeMillis();
+ io.netty.handler.codec.http2.DefaultHttp2Headers heartbeat =
+ new io.netty.handler.codec.http2.DefaultHttp2Headers();
+
+ // Add standard heartbeat headers
+ heartbeat.set("tri-heartbeat", String.valueOf(now));
+ heartbeat.set("tri-last-acked", String.valueOf(getLastAckedSeq()));
+ heartbeat.set("tri-state", heartbeatState.name().toLowerCase());
+
+ // Add custom headers
+ if (customHeaders != null) {
+ for (java.util.Map.Entry<CharSequence, Object> entry : customHeaders.entrySet()) {
+ heartbeat.set(entry.getKey(), String.valueOf(entry.getValue()));
+ }
+ }
+
+ HeaderQueueCommand cmd =
+ HeaderQueueCommand.createHeaders(getCurrentStreamChannelFuture(), heartbeat, false);
+ writeQueue.enqueueFuture(cmd, parent.eventLoop()).addListener(future -> {
+ if (!future.isSuccess()) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Failed to send heartbeat with custom headers", future.cause());
+ } else {
+ LOGGER.debug("Successfully sent heartbeat with custom headers");
+ }
+ });
+
+ lastHeartbeatSentTime = now;
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Error sending heartbeat with headers", e);
+ }
+ }
+
+ private final Object stateTransitionLock = new Object();
+
+ private void transitionToState(HeartbeatState newState, String reason) {
+ synchronized (stateTransitionLock) {
+ HeartbeatState oldState = heartbeatState;
+
+ // Prevent state transitions once resources are cleaned or CLOSED
+ if (oldState == HeartbeatState.CLOSED || reliabilityResourcesCleaned) {
+ LOGGER.debug(
+ "Ignoring state transition from {} to {} ({}) - resources cleaned: {}",
+ oldState,
+ newState,
+ reason,
+ reliabilityResourcesCleaned);
+ return;
+ }
+
+ // Validate state transition logic
+ if (!isValidStateTransition(oldState, newState)) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Invalid state transition from " + oldState + " to " + newState + " (" + reason
+ + ") for session: " + sessionId);
+ return;
+ }
+
+ heartbeatState = newState;
+
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Heartbeat state transition: " + oldState + " -> " + newState + " (" + reason + "), sessionId: "
+ + sessionId);
+
+ // Trigger state change callbacks
+ updateReliabilityState(newState.name(), reason);
+ }
+ }
+
+ private boolean isValidStateTransition(HeartbeatState from, HeartbeatState to) {
+ // Define valid state transitions
+ switch (from) {
+ case HEALTHY:
+ return to == HeartbeatState.SUSPECT
+ || to == HeartbeatState.FAILED
+ || to == HeartbeatState.PAUSED
+ || to == HeartbeatState.CLOSED;
+ case SUSPECT:
+ return to == HeartbeatState.HEALTHY
+ || to == HeartbeatState.RECONNECTING
+ || to == HeartbeatState.FAILED
+ || to == HeartbeatState.PAUSED
+ || to == HeartbeatState.CLOSED;
+ case RECONNECTING:
+ return to == HeartbeatState.HEALTHY
+ || to == HeartbeatState.FAILED
+ || to == HeartbeatState.PAUSED
+ || to == HeartbeatState.CLOSED;
+ case FAILED:
+ return to == HeartbeatState.RECONNECTING
+ || to == HeartbeatState.HEALTHY
+ || to == HeartbeatState.PAUSED
+ || to == HeartbeatState.CLOSED; // Allow recovery from FAILED state
+ case PAUSED:
+ return to == HeartbeatState.RECONNECTING
+ || to == HeartbeatState.HEALTHY
+ || to == HeartbeatState.FAILED
+ || to == HeartbeatState.CLOSED; // Allow recovery from PAUSED state
+ case CLOSED:
+ return false; // CLOSED is terminal
+ default:
+ return false;
+ }
+ }
+
+ private void attemptReconnect() {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Connection reconnection needed for session: " + sessionId);
+
+ // Schedule recovery in dedicated recovery executor to avoid blocking heartbeat thread
+ try {
+ recoveryExecutor.execute(() -> {
+ try {
+ performSessionRecovery();
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Session recovery failed for session: " + sessionId, e);
+ transitionToState(HeartbeatState.FAILED, "Session recovery failed");
+ }
+ });
+ } catch (java.util.concurrent.RejectedExecutionException e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Failed to schedule session recovery for session: " + sessionId);
+ transitionToState(HeartbeatState.FAILED, "Unable to schedule recovery");
+ }
+ }
+
+ private void performSessionRecovery() {
+ if (heartbeatState == HeartbeatState.CLOSED) {
+ return;
+ }
+
+ // Handle PAUSED state recovery first
+ if (heartbeatState == HeartbeatState.PAUSED) {
+ LOGGER.info(
+ "Detected PAUSED state during session recovery, attempting to recover for session: {}", sessionId);
+ recoverFromPausedState();
+ return;
+ }
+
+ // Handle FAILED state recovery
+ if (heartbeatState == HeartbeatState.FAILED) {
+ LOGGER.info(
+ "Detected FAILED state during session recovery, attempting to recover for session: {}", sessionId);
+ recoverFromFailedState();
+ return;
+ }
+
+ LOGGER.info(
+ "Starting session recovery for session: {}, pending messages: {}",
+ sessionId,
+ reliabilityEnabled && pendingMessages != null ? pendingMessages.size() : 0);
+
+ // Check if connection is still active
+ if (!parent.isActive()) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Connection is inactive for session: " + sessionId + ", attempting reconnection");
+
+ // Attempt real reconnection using reconnection manager
+ if (reconnectionManager != null) {
+ attemptRealReconnection();
+ return;
+ } else {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "No reconnection manager available for session: " + sessionId + ", recovery failed");
+ transitionToState(HeartbeatState.FAILED, "No reconnection manager available");
+ return;
+ }
+ }
+
+ // Connection is active, perform session recovery
+ performActiveSessionRecovery();
+ }
+
+ private void attemptRealReconnection() {
+ LOGGER.info("Attempting real reconnection for session: {}", sessionId);
+
+ reconnectionManager.attemptReconnection().whenComplete((success, throwable) -> {
+ if (throwable != null) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Reconnection failed for session: " + sessionId, throwable);
+ handleReconnectionFailure("Reconnection attempt failed with exception", throwable);
+ return;
+ }
+
+ if (success && reconnectionManager.isConnectionActive()) {
+ LOGGER.info("Reconnection succeeded for session: {}", sessionId);
+
+ try {
+ // After successful reconnection, the underlying connection is reestablished
+ // The existing stream can continue to work with the new connection
+ // since the connection client will provide the new channel transparently
+ performActiveSessionRecovery();
+ LOGGER.info("Session recovery completed after reconnection for session: {}", sessionId);
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Error during session recovery after reconnection for session: " + sessionId,
+ e);
+ handleReconnectionFailure("Error during session recovery", e);
+ }
+ } else {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Reconnection failed for session: " + sessionId);
+ handleReconnectionFailure("Reconnection failed - no active connection", null);
+ }
+ });
+ }
+
+ private void handleReconnectionFailure(String reason, Throwable throwable) {
+ try {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Handling reconnection failure for session: " + sessionId + " - Reason: " + reason,
+ throwable);
+
+ // Mark the session as failed
+ transitionToState(HeartbeatState.FAILED, reason);
+
+ // Notify the business layer of the connection failure
+ notifyBusinessLayerOfFailure(reason, throwable);
+
+ // Pause reliability instead of permanent cleanup to allow recovery
+ pauseReliability();
+
+ // Log failure statistics
+ logReconnectionFailureStatistics();
+
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR, "", "", "Error during reconnection failure handling for session: " + sessionId, e);
+ }
+ }
+
+ /**
+ * Notify business layer of connection failure.
+ * For recoverable failures, only trigger state change callback (stream continues).
+ * For non-recoverable failures, call onComplete to terminate the stream.
+ */
+ private void notifyBusinessLayerOfFailure(String reason, Throwable throwable) {
+ try {
+ // Create error status for classification
+ TriRpcStatus errorStatus;
+ if (throwable != null) {
+ errorStatus = TriRpcStatus.INTERNAL.withDescription(
+ "Reliable stream connection failed: " + reason + ": " + throwable.getMessage());
+ } else {
+ errorStatus = TriRpcStatus.INTERNAL.withDescription("Reliable stream connection failed: " + reason);
+ }
+
+ // Check if error is recoverable
+ boolean recoverable =
+ (throwable != null) ? isRecoverableException(throwable) : isRecoverableError(errorStatus);
+
+ if (recoverable) {
+ // Recoverable failure: Only notify state change, stream continues for recovery
+ LOGGER.info(
+ "Recoverable connection failure for session: {} - Reason: {}, triggering state callback",
+ sessionId,
+ reason);
+
+ // Notify via state change callback (if registered)
+ if (stateChangeCallback != null) {
+ try {
+ stateChangeCallback.accept(heartbeatState.name());
+ } catch (Exception e) {
+ LOGGER.warn(
+ INTERNAL_ERROR, "", "", "Error in state change callback for session: " + sessionId, e);
+ }
+ }
+
+ // Do NOT call onComplete - stream remains alive for recovery
+ LOGGER.debug(
+ "Stream continues for session: {} after recoverable failure, state: {}",
+ sessionId,
+ heartbeatState);
+
+ } else {
+ // Non-recoverable failure: Terminate the stream
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Non-recoverable connection failure for session: " + sessionId + " - Reason: " + reason
+ + ", terminating stream");
+
+ if (listener != null) {
+ // Call onComplete to terminate the stream
+ listener.onComplete(errorStatus, new java.util.HashMap<>());
+ LOGGER.debug(
+ "Successfully terminated stream for session: {} due to non-recoverable error", sessionId);
+ } else {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "No listener available to terminate stream for session: " + sessionId);
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Error notifying business layer of connection failure for session: " + sessionId,
+ e);
+ }
+ }
+
+ private void logReconnectionFailureStatistics() {
+ try {
+ if (reliabilityEnabled) {
+ long totalSent = sequenceNumber != null ? sequenceNumber.get() : 0;
+ int pendingCount = pendingMessages != null ? pendingMessages.size() : 0;
+ long totalRetryCount = getTotalRetryCount();
+
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Reconnection failure statistics for session: " + sessionId + " - " + "Total messages sent: "
+ + totalSent + ", Pending messages: " + pendingCount + ", Total retries: "
+ + totalRetryCount);
+ }
+ } catch (Exception e) {
+ LOGGER.debug("Error logging reconnection failure statistics for session: {}", sessionId, e);
+ }
+ }
+
+ private void performActiveSessionRecovery() {
+ // 从持久化存储恢复消息(重连后可能需要重新加载)
+ recoverPendingMessages();
+
+ // Reset connection state
+ lastHeartbeatAckTime = System.currentTimeMillis();
+ missedHeartbeats = 0;
+ transitionToState(HeartbeatState.HEALTHY, "Session recovery successful");
+
+ // Notify business layer that session has been recovered
+ notifyRecovery("Session reconnected and recovered successfully");
+
+ // Update stream channel after successful reconnection (fix pseudo-reconnection issue)
+ updateStreamChannelAfterReconnection();
+
+ // CRITICAL: Restart reliability schedulers after successful reconnection
+ restartReliabilitySchedulers();
+
+ // Reset temporary failure count after successful recovery
+ resetTemporaryFailureCount();
+
+ // Send a heartbeat to verify connection
+ sendHeartbeatIfNeeded();
+
+ LOGGER.info("Session recovery completed for session: {}", sessionId);
+ }
+
+ /**
+ * Restart reliability schedulers after successful reconnection.
+ * This ensures heartbeat and retry mechanisms continue to work after reconnection.
+ */
+ private void restartReliabilitySchedulers() {
+ try {
+ LOGGER.info("Restarting reliability schedulers after reconnection for session: {}", sessionId);
+
+ // Check if heartbeat scheduler needs restarting
+ if (heartbeatScheduler == null || heartbeatScheduler.isShutdown()) {
+ LOGGER.info("Recreating heartbeat scheduler for session: {}", sessionId);
+ heartbeatScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-heartbeat-" + sessionId);
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ // Check if retry scheduler needs restarting
+ if (retryScheduler == null || retryScheduler.isShutdown()) {
+ LOGGER.info("Recreating retry scheduler for session: {}", sessionId);
+ retryScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-retry-" + sessionId);
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ // Check if recovery executor needs restarting
+ if (recoveryExecutor == null || recoveryExecutor.isShutdown()) {
+ LOGGER.info("Recreating recovery executor for session: {}", sessionId);
+ recoveryExecutor = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-recovery-" + sessionId);
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ // Restart heartbeat scheduling
+ scheduleHeartbeat();
+
+ // Restart retry scheduling
+ scheduleRetryCheck();
+
+ LOGGER.info("Successfully restarted reliability schedulers for session: {}", sessionId);
+
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR, "", "", "Failed to restart reliability schedulers for session: " + sessionId, e);
+ // Don't fail the recovery process, but log the error
+ }
+ }
+
+ private void updateStreamChannelAfterReconnection() {
+ try {
+ LOGGER.info("Starting real channel replacement after reconnection for session: {}", sessionId);
+
+ // Use the new type-safe method to get stream future
+ TripleStreamChannelFuture newStreamChannelFuture = reconnectionManager.createStreamFuture();
+ if (newStreamChannelFuture == null) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "No new stream channel available after reconnection for session: " + sessionId);
+ handleReconnectionFailure("Failed to create new stream channel", null);
+ return;
+ }
+
+ // CRITICAL FIX: Wait for the new stream channel to be actually created before proceeding
+ try {
+ LOGGER.debug("Waiting for new stream channel creation to complete for session: {}", sessionId);
+ // Wait for the TripleStreamChannelFuture to complete with a reasonable timeout
+ Object newStreamChannel = newStreamChannelFuture.get(10, java.util.concurrent.TimeUnit.SECONDS);
+
+ if (newStreamChannel == null) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "New stream channel creation completed but returned null for session: " + sessionId);
+ handleReconnectionFailure("New stream channel is null after creation", null);
+ return;
+ }
+
+ LOGGER.info(
+ "New stream channel created successfully for session: {}, proceeding with replacement",
+ sessionId);
+
+ } catch (java.util.concurrent.TimeoutException e) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Timeout waiting for new stream channel creation for session: " + sessionId,
+ e);
+ handleReconnectionFailure("New stream channel creation timeout", e);
+ return;
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Error waiting for new stream channel creation for session: " + sessionId,
+ e);
+ handleReconnectionFailure("New stream channel creation failed", e);
+ return;
+ }
+
+ // Now perform the actual replacement in a synchronous manner
+ try {
+ // Use atomic switch to ensure thread safety
+ // This is critical to ensure new messages use the reconnected channel
+ boolean switchSuccess = switchStreamChannelAtomically(newStreamChannelFuture);
+
+ if (switchSuccess) {
+ LOGGER.info(
+ "Successfully replaced stream channel future for session: {} after reconnection",
+ sessionId);
+ } else {
+ throw new RuntimeException("Atomic stream channel switch failed");
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR, "", "", "Error during stream channel replacement for session: " + sessionId, e);
+ handleReconnectionFailure("Stream channel replacement failed", e);
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Error during stream channel update for session: " + sessionId, e);
+ handleReconnectionFailure("Stream channel update failed", e);
+ }
+ }
+
+ private void cleanupOldConnectionResources() {
+ try {
+ // Clear any old pending network operations
+ // The old transport listener will naturally be replaced when the new channel is active
+ LOGGER.debug("Cleaned up old connection resources for session: {}", sessionId);
+ } catch (Exception e) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Error during old connection resource cleanup for session: " + sessionId,
+ e);
+ }
+ }
+
+ /**
+ * Recover from FAILED state by attempting reconnection and resource restoration.
+ * This method provides a recovery path from permanent failure state.
+ * Enhanced to handle complete resource reinitialization even after cleanup.
+ */
+ private boolean recoverFromFailedState() {
+ try {
+ LOGGER.info("Starting comprehensive recovery from FAILED state for session: {}", sessionId);
+
+ // Always attempt to fully reinitialize reliability infrastructure
+ return performComprehensiveRecovery("Manual recovery from FAILED state");
+
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR, "", "", "Error during recovery from FAILED state for session: " + sessionId, e);
+ transitionToState(HeartbeatState.FAILED, "Recovery attempt failed with exception");
+ return false;
+ }
+ }
+
+ /**
+ * Comprehensive recovery that can restore from any failure state, including after cleanup.
+ * This method completely reinitializes the reliability infrastructure.
+ */
+ private boolean performComprehensiveRecovery(String reason) {
+ try {
+ LOGGER.info("Starting comprehensive recovery for session: {} - Reason: {}", sessionId, reason);
+
+ // Step 1: Transition to RECONNECTING state
+ HeartbeatState previousState = heartbeatState;
+ transitionToState(HeartbeatState.RECONNECTING, reason);
+ if (heartbeatState != HeartbeatState.RECONNECTING) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to transition from " + previousState + " to RECONNECTING state for session: "
+ + sessionId);
+ return false;
+ }
+
+ // Step 2: Reset all cleanup flags
+ reliabilityResourcesCleaned = false;
+ reliabilityEnabled = true;
+
+ // Step 3: Reinitialize all executors
+ reinitializeExecutors();
+
+ // Step 4: Reinitialize pending message structure
+ if (pendingMessages == null) {
+ pendingMessages = new ConcurrentHashMap<>();
+ LOGGER.info("Reinitialized pendingMessages map for session: {}", sessionId);
+ }
+
+ // Step 5: Reinitialize messageStore and recover data
+ boolean storeRecovered = reinitializeMessageStore();
+ if (!storeRecovered) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "MessageStore reinitializtion failed, continuing without persistence for session: "
+ + sessionId);
+ }
+
+ // Step 6: Reset sequence tracking
+ if (sequenceNumber == null) {
+ sequenceNumber = new AtomicLong(0);
+ }
+ if (lastAckedSeq == null) {
+ java.lang.reflect.Field lastAckedSeqField =
+ AbstractTripleClientStream.class.getDeclaredField("lastAckedSeq");
+ lastAckedSeqField.setAccessible(true);
+ lastAckedSeqField.set(this, new AtomicLong(0));
+ }
+
+ // Step 7: Reset counters
+ if (inFlightCount == null) {
+ java.lang.reflect.Field inFlightCountField =
+ AbstractTripleClientStream.class.getDeclaredField("inFlightCount");
+ inFlightCountField.setAccessible(true);
+ inFlightCountField.set(this, new AtomicInteger(0));
+ }
+
+ // Step 8: Load pending messages from persistent storage
+ loadPendingMessagesFromStorage();
+
+ // Step 9: Renegotiate ACK starting points after reconnection
+ renegotiateAckPoints();
+
+ // Step 10: Restart reliability scheduling
+ scheduleHeartbeat();
+ scheduleRetryCheck();
+
+ // Step 11: Attempt the actual reconnection
+ attemptRealReconnection();
+
+ LOGGER.info("Successfully completed comprehensive recovery for session: {}", sessionId);
+ return true;
+
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Error during comprehensive recovery for session: " + sessionId, e);
+ transitionToState(HeartbeatState.FAILED, "Comprehensive recovery failed");
+ return false;
+ }
+ }
+
+ /**
+ * Initialize or reinitialize all executor services for reliability.
+ */
+ private void reinitializeExecutors() {
+ try {
+ // Recreate heartbeat scheduler if needed
+ if (heartbeatScheduler == null || heartbeatScheduler.isShutdown()) {
+ LOGGER.info("Recreating heartbeat scheduler for session: {}", sessionId);
+ heartbeatScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-heartbeat-" + sessionId);
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ // Recreate retry scheduler if needed
+ if (retryScheduler == null || retryScheduler.isShutdown()) {
+ LOGGER.info("Recreating retry scheduler for session: {}", sessionId);
+ retryScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-retry-" + sessionId);
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ // Recreate recovery executor if needed
+ if (recoveryExecutor == null || recoveryExecutor.isShutdown()) {
+ LOGGER.info("Recreating recovery executor for session: {}", sessionId);
+ recoveryExecutor = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "reliability-recovery-" + sessionId);
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ LOGGER.info("Successfully reinitialized all executors for session: {}", sessionId);
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Error reinitializing executors for session: " + sessionId, e);
+ throw new RuntimeException("Executor reinitialization failed", e);
+ }
+ }
+
+ /**
+ * Reinitialize messageStore and ensure it's ready for use.
+ */
+ private boolean reinitializeMessageStore() {
+ try {
+ // Close existing store if it exists
+ if (messageStore != null) {
+ try {
+ SessionMetadata existingMetadata = new SessionMetadata(
+ sessionId, url.getServiceKey(), System.currentTimeMillis(), url, config);
+ messageStore.close(existingMetadata);
+ } catch (Exception e) {
+ LOGGER.debug("Error closing existing messageStore during reinit for session: {}", sessionId, e);
+ }
+ }
+
+ // Create new messageStore
+ LOGGER.info("Reinitializing messageStore for session: {}", sessionId);
+ String storeType = url.getParameter("tri.client.pending.store.type", "memory");
+ messageStore = ExtensionLoader.getExtensionLoader(PendingMessageStore.class)
+ .getExtension(storeType);
+
+ // Initialize with fresh session metadata
+ SessionMetadata metadata =
+ new SessionMetadata(sessionId, url.getServiceKey(), System.currentTimeMillis(), url, config);
+ messageStore.init(metadata);
+
+ LOGGER.info("Successfully reinitialized messageStore for session: {}", sessionId);
+ return true;
+
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Failed to reinitialize messageStore for session: " + sessionId, e);
+ messageStore = null;
+ return false;
+ }
+ }
+
+ /**
+ * Recover from PAUSED state by restoring schedulers and continuing with normal recovery.
+ * This method provides a recovery path from temporary pause state without full reinitialization.
+ */
+ private boolean recoverFromPausedState() {
+ try {
+ LOGGER.info("Starting recovery from PAUSED state for session: {}", sessionId);
+
+ // Verify we are in PAUSED state
+ if (heartbeatState != HeartbeatState.PAUSED) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Cannot recover from PAUSED state - current state is: " + heartbeatState + " for session: "
+ + sessionId);
+ return false;
+ }
+
+ // Use comprehensive recovery which handles all edge cases
+ return performComprehensiveRecovery("Recovery from PAUSED state");
+
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR, "", "", "Error during recovery from PAUSED state for session: " + sessionId, e);
+ transitionToState(HeartbeatState.FAILED, "PAUSED state recovery failed with exception");
+ return false;
+ }
+ }
+
+ /**
+ * Load pending messages from persistent storage during recovery.
+ * This method restores messages that were persisted before connection failure.
+ */
+ private void loadPendingMessagesFromStorage() {
+ try {
+ if (messageStore == null) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Cannot load pending messages - messageStore is null for session: " + sessionId);
+ return;
+ }
+
+ LOGGER.info("Loading pending messages from persistent storage for session: {}", sessionId);
+
+ // Retrieve all pending messages from the store
+ java.util.List<org.apache.dubbo.rpc.protocol.tri.store.PendingMessage> storedMessages =
+ messageStore.load(sessionId);
+
+ if (storedMessages == null || storedMessages.isEmpty()) {
+ LOGGER.info("No pending messages found in storage for session: {}", sessionId);
+ return;
+ }
+
+ // Initialize pendingMessages if needed
+ if (pendingMessages == null) {
+ pendingMessages = new java.util.concurrent.ConcurrentHashMap<>();
+ } else {
+ pendingMessages.clear(); // Clear any existing messages
+ }
+
+ // Restore messages to in-memory map, converting from store format to runtime format
+ int loadedCount = 0;
+ for (org.apache.dubbo.rpc.protocol.tri.store.PendingMessage storedMsg : storedMessages) {
+ // Convert from store.PendingMessage to runtime PendingMessage
+ PendingMessage runtimeMsg = new PendingMessage(
+ storedMsg.getSequence(),
+ storedMsg.getMessage(),
+ storedMsg.getCompressFlag(),
+ storedMsg.getSentTime(),
+ storedMsg.getRetryCount());
+
+ // Set additional runtime fields if they exist
+ if (storedMsg.getFirstSendTime() > 0) {
+ runtimeMsg.firstSendTime = storedMsg.getFirstSendTime();
+ }
+ if (storedMsg.getNextRetryDelay() > 0) {
+ runtimeMsg.nextRetryDelay = storedMsg.getNextRetryDelay();
+ }
+
+ pendingMessages.put(runtimeMsg.getSequence(), runtimeMsg);
+ loadedCount++;
+ }
+
+ LOGGER.info("Successfully loaded {} pending messages from storage for session: {}", loadedCount, sessionId);
+
+ // Update sequence number to the highest loaded sequence
+ if (!storedMessages.isEmpty()) {
+ long maxSeq = storedMessages.stream()
+ .mapToLong(org.apache.dubbo.rpc.protocol.tri.store.PendingMessage::getSequence)
+ .max()
+ .orElse(0);
+ if (maxSeq > sequenceNumber.get()) {
+ sequenceNumber.set(maxSeq);
+ LOGGER.info(
+ "Updated sequence number to {} after loading messages for session: {}", maxSeq, sessionId);
+ }
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR, "", "", "Error loading pending messages from storage for session: " + sessionId, e);
+ // Don't fail recovery just because of storage loading issues
+ // The connection can still work, just without previously pending messages
+ }
+ }
+
+ /**
+ * Renegotiate ACK starting points after reconnection.
+ * This helps establish a common understanding with the server about message acknowledgments.
+ */
+ private void renegotiateAckPoints() {
+ try {
+ LOGGER.info("Starting ACK point renegotiation for session: {}", sessionId);
+
+ // Get the last acknowledged sequence number
+ long lastAcked = lastAckedSeq.get();
+ long currentSeq = sequenceNumber.get();
+
+ LOGGER.info(
+ "ACK negotiation for session: {} - lastAcked: {}, currentSeq: {}",
+ sessionId,
+ lastAcked,
+ currentSeq);
+
+ // Send a heartbeat with current state to re-establish ACK baseline
+ if (reliabilityEnabled && heartbeatState == HeartbeatState.RECONNECTING) {
+ // Create a special recovery heartbeat that includes sequence info
+ java.util.Map<CharSequence, Object> recoveryHeaders = new java.util.HashMap<>();
+ recoveryHeaders.put("tri-header-last-ack", String.valueOf(lastAcked));
+ recoveryHeaders.put("tri-header-current-seq", String.valueOf(currentSeq));
+ recoveryHeaders.put("tri-header-recovery-mark", "true");
+
+ // Send the recovery heartbeat
+ sendHeartbeatWithHeaders(recoveryHeaders);
+
+ LOGGER.info(
+ "Sent recovery heartbeat for ACK renegotiation, session: {}, lastAck: {}, currentSeq: {}",
+ sessionId,
+ lastAcked,
+ currentSeq);
+ }
+
+ // Reset missed heartbeats since we're starting fresh
+ missedHeartbeats = 0;
+ lastHeartbeatSentTime = System.currentTimeMillis();
+
+ LOGGER.info("Successfully completed ACK point renegotiation for session: {}", sessionId);
+
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Error during ACK point renegotiation for session: " + sessionId, e);
+ // Don't fail recovery just because of ACK negotiation issues
+ // The stream can still work, just with potentially some duplicate messages
+ }
+ }
+
+ private void resendPendingMessages() {
+ // Check if reliability is enabled and pendingMessages is initialized
+ if (!reliabilityEnabled || pendingMessages == null || pendingMessages.isEmpty()) {
+ return;
+ }
+
+ LOGGER.info(
+ "Resending {} pending messages for session: {} using new connection",
+ pendingMessages.size(),
+ sessionId);
+
+ // Verify that we have a valid new connection before resending
+ if (!validateNewConnectionForResend()) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Cannot resend messages - no valid new connection available for session: " + sessionId);
+ handleConnectionFailure();
+ return;
+ }
+
+ // Create a copy to avoid concurrent modification
+ java.util.List<PendingMessage> messagesToResend = new java.util.ArrayList<>(pendingMessages.values());
+
+ // IMPORTANT: Filter out messages that exceeded retry limits
+ // This prevents infinite retry on repeated reconnections
+ int originalCount = messagesToResend.size();
+ messagesToResend.removeIf(pending -> {
+ if (!isStillRetryable(pending)) {
+ // Remove from pending messages map
+ pendingMessages.remove(pending.getSequence());
+ // Decrement InFlight count for dropped message
+ inFlightCount.decrementAndGet();
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Dropping message seq=" + pending.getSequence()
+ + " during reconnection resend (retry limit exceeded)");
+ return true; // Remove from resend list
+ }
+ return false; // Keep for resend
+ });
+
+ int droppedCount = originalCount - messagesToResend.size();
+ if (droppedCount > 0) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Dropped " + droppedCount + " messages during reconnection resend due to retry limits for session: "
+ + sessionId);
+ }
+
+ if (messagesToResend.isEmpty()) {
+ LOGGER.info("No retryable messages remaining after filtering for session: {}", sessionId);
+ return;
+ }
+
+ // Sort by sequence number to maintain order
+ messagesToResend.sort((a, b) -> Long.compare(a.getSequence(), b.getSequence()));
+
+ // Update retry counts for all messages to be resent
+ updateRetryCountsForResend(messagesToResend);
+
+ // Schedule messages asynchronously with delays to avoid overwhelming the connection
+ scheduleMessagesWithDelay(messagesToResend, 0);
+ }
+
+ private boolean validateNewConnectionForResend() {
+ try {
+ // Check if streamChannelFuture is valid and ready
+ TripleStreamChannelFuture currentFuture = getCurrentStreamChannelFuture();
+ if (currentFuture == null || currentFuture.isCancelled() || currentFuture.isCompletedExceptionally()) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Stream channel future is not valid for resend - null: " + (currentFuture == null)
+ + ", cancelled: " + (currentFuture != null && currentFuture.isCancelled())
+ + ", exceptional: "
+ + (currentFuture != null && currentFuture.isCompletedExceptionally()));
+ return false;
+ }
+
+ // Check if parent channel is active
+ if (parent == null || !parent.isActive()) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Parent channel is not active for resend - null: " + (parent == null) + ", active: "
+ + (parent != null && parent.isActive()));
+ return false;
+ }
+
+ // Check if write queue is available
+ if (writeQueue == null) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Write queue is not available for resend");
+ return false;
+ }
+
+ LOGGER.debug("New connection validated successfully for resend on session: {}", sessionId);
+ return true;
+ } catch (Exception e) {
+ LOGGER.warn(
+ INTERNAL_ERROR, "", "", "Error validating new connection for resend on session: " + sessionId, e);
+ return false;
+ }
+ }
+
+ private void updateRetryCountsForResend(java.util.List<PendingMessage> messages) {
+ try {
+ for (PendingMessage pending : messages) {
+ // Increment retry count to track reconnection-triggered retries
+ pending.incrementRetry();
+ LOGGER.debug(
+ "Updated retry count for message seq: {} to {} for session: {}",
+ pending.getSequence(),
+ pending.getRetryCount(),
+ sessionId);
+ }
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error updating retry counts for resend on session: " + sessionId, e);
+ }
+ }
+
+ private void scheduleMessagesWithDelay(java.util.List<PendingMessage> messages, int currentIndex) {
+ if (currentIndex >= messages.size()) {
+ LOGGER.info("Completed resending pending messages for session: {}", sessionId);
+ return;
+ }
+
+ PendingMessage pending = messages.get(currentIndex);
+
+ try {
+ // Verify that we still have a valid connection before resending
+ if (!validateNewConnectionForResend()) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Connection is no longer valid during message resend for session: " + sessionId);
+ handleConnectionFailure();
+ return;
+ }
+
+ // Reset the sent time to current time for retry logic
+ pending.updateSentTime(System.currentTimeMillis());
+
+ // Resend the message
+ retryMessage(pending);
+
+ LOGGER.debug("Resent message with sequence: {} for session: {}", pending.getSequence(), sessionId);
+
+ // Schedule next message with 10ms delay
+ if (retryScheduler != null && !retryScheduler.isShutdown()) {
+ retryScheduler.schedule(
+ () -> scheduleMessagesWithDelay(messages, currentIndex + 1), 10, TimeUnit.MILLISECONDS);
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to resend message with sequence: " + pending.getSequence() + " for session: " + sessionId,
+ e);
+
+ // If we fail to resend a message after reconnection, treat it as a permanent failure
+ handleConnectionFailure();
+
+ // Do not continue with next messages - connection has failed
+ return;
+ }
+ }
+
+ private void handleConnectionFailure() {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Connection failed temporarily for session: " + sessionId + ", will allow recovery");
+
+ // Notify business layer of the connection failure
+ notifyBusinessLayerOfFailure("Connection failed during message resend", null);
+
+ // Handle as temporary failure to allow recovery
+ handleTemporaryFailureWithCount("Connection failure during resend");
+ }
+
+ private void onHeartbeatAck() {
+ lastHeartbeatAckTime = System.currentTimeMillis();
+ missedHeartbeats = 0; // Reset missed heartbeat counter
+
+ // Transition back to healthy state if not already
+ if (heartbeatState != HeartbeatState.HEALTHY) {
+ HeartbeatState previousState = heartbeatState;
+ transitionToState(HeartbeatState.HEALTHY, "Heartbeat ACK received");
+
+ // Notify recovery if we were in a problematic state
+ if (previousState == HeartbeatState.SUSPECT || previousState == HeartbeatState.RECONNECTING) {
+ notifyRecovery("Heartbeat connectivity restored from " + previousState + " state");
+ // Reset failure count when connectivity is restored
+ resetTemporaryFailureCount();
+ }
+ }
+ }
+
+ private void sendHeartbeatAckToServer(String originalHeartbeat) {
+ try {
+ if (!parent.isActive()) {
+ return;
+ }
+
+ io.netty.handler.codec.http2.DefaultHttp2Headers heartbeatAck =
+ new io.netty.handler.codec.http2.DefaultHttp2Headers();
+ heartbeatAck.set("tri-heartbeat-ack", originalHeartbeat);
+ heartbeatAck.set("tri-client-time", String.valueOf(System.currentTimeMillis()));
+
+ HeaderQueueCommand cmd =
+ HeaderQueueCommand.createHeaders(getCurrentStreamChannelFuture(), heartbeatAck, false);
+ writeQueue.enqueue(cmd);
+
+ LOGGER.debug("Sent heartbeat ACK to server: {}", originalHeartbeat);
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Failed to send heartbeat ACK to server", e);
+ }
+ }
+
+ private void scheduleRetryCheck() {
+ if (!reliabilityEnabled
+ || retryScheduler == null
+ || retryScheduler.isShutdown()
+ || heartbeatState == HeartbeatState.CLOSED) {
+ return;
+ }
+
+ try {
+ // Cancel existing retry task if any
+ if (currentRetryTask != null && !currentRetryTask.isDone()) {
+ currentRetryTask.cancel(false);
+ LOGGER.debug("Cancelled previous retry check task for session: {}", sessionId);
+ }
+
+ // Use single delayed scheduling instead of fixed rate to avoid duplicate scheduling
+ currentRetryTask = scheduleTrackedTask(
+ retryScheduler,
+ () -> {
+ try {
+ long now = System.currentTimeMillis();
+ processPendingRetries(now);
+ // Schedule next check with adaptive delay
+ scheduleNextRetryCheck();
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Retry check error for session: " + sessionId, e);
+ }
+ },
+ 1000,
+ TimeUnit.MILLISECONDS);
+
+ if (currentRetryTask != null) {
+ LOGGER.debug("Scheduled new retry check task for session: {}", sessionId);
+ }
+ } catch (java.util.concurrent.RejectedExecutionException e) {
+ LOGGER.debug("Failed to schedule retry check - scheduler shut down for session: {}", sessionId);
+ }
+ }
+
+ /**
+ * Schedule the next retry check with adaptive delay based on pending messages.
+ * This avoids duplicate scheduling and provides better resource utilization.
+ */
+ private void scheduleNextRetryCheck() {
+ if (!reliabilityEnabled
+ || retryScheduler == null
+ || retryScheduler.isShutdown()
+ || heartbeatState == HeartbeatState.CLOSED) {
+ return;
+ }
+
+ try {
+ // Calculate adaptive delay based on the earliest pending retry time
+ long nextDelay = calculateOptimalRetryCheckDelay();
+
+ // No need to update currentRetryTask here since this is called from within an existing retry task
+ ScheduledFuture<?> nextRetryTask = scheduleTrackedTask(
+ retryScheduler,
+ () -> {
+ try {
+ long now = System.currentTimeMillis();
+ processPendingRetries(now);
+ // Schedule next check recursively
+ scheduleNextRetryCheck();
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Retry check error for session: " + sessionId, e);
+ }
+ },
+ nextDelay,
+ TimeUnit.MILLISECONDS);
+
+ if (nextRetryTask != null) {
+ LOGGER.debug("Scheduled next retry check with delay: {}ms for session: {}", nextDelay, sessionId);
+ }
+ } catch (java.util.concurrent.RejectedExecutionException e) {
+ LOGGER.debug("Failed to schedule next retry check - scheduler shut down for session: {}", sessionId);
+ }
+ }
+
+ /**
+ * Calculate the optimal delay for the next retry check.
+ * Returns the time until the next pending message should be retried.
+ */
+ private long calculateOptimalRetryCheckDelay() {
+ if (pendingMessages.isEmpty()) {
+ return 1000; // Default 1 second when no pending messages
+ }
+
+ long now = System.currentTimeMillis();
+ long minDelay = Long.MAX_VALUE;
+
+ // Find the earliest retry time among all pending messages
+ for (PendingMessage pending : pendingMessages.values()) {
+ long nextRetryTime = pending.getSentTime() + config.getRetryTimeout();
+ if (nextRetryTime > now) {
+ long delay = nextRetryTime - now;
+ minDelay = Math.min(minDelay, delay);
+ } else {
+ // Message is already overdue, check immediately
+ return 100; // Small delay for immediate processing
+ }
+ }
+
+ // Ensure minimum delay of 100ms and maximum of 1 second
+ return Math.max(100, Math.min(1000, minDelay));
+ }
+
+ /**
+ * Process all pending retries with unified timing management.
+ * This avoids duplicate scheduling and ensures consistent retry behavior.
+ */
+ private void processPendingRetries(long now) {
+ if (pendingMessages.isEmpty()) {
+ return;
+ }
+
+ // Create a copy to avoid concurrent modification during iteration
+ java.util.List<PendingMessage> messagesToCheck = new java.util.ArrayList<>(pendingMessages.values());
+
+ for (PendingMessage pending : messagesToCheck) {
+ // Check if this message needs retry
+ if (now - pending.getSentTime() > config.getRetryTimeout()) {
+ checkAndRetryWithUnifiedTiming(pending, now);
+ }
+ }
+ }
+
+ /**
+ * Legacy method for backward compatibility.
+ * Use checkAndRetryWithUnifiedTiming() for new unified timing management.
+ */
+ private void checkAndRetry(PendingMessage pending, long now) {
+ checkAndRetryWithUnifiedTiming(pending, now);
+ }
+
+ /**
+ * Check and retry with unified timing management.
+ * This avoids duplicate scheduling and ensures consistent retry behavior.
+ */
+ private void checkAndRetryWithUnifiedTiming(PendingMessage pending, long now) {
+ // 检查最大重试次数
+ if (pending.getRetryCount() >= config.getMaxRetries()) {
+ pendingMessages.remove(pending.getSequence());
+ // Decrement InFlight count since we're giving up on this message
+ inFlightCount.decrementAndGet();
+ cancelByLocal(TriRpcStatus.DEADLINE_EXCEEDED.withDescription(
+ "Max retries exceeded for seq: " + pending.getSequence()));
+ return;
+ }
+
+ // 检查总重试超时
+ long totalElapsedTime = now - pending.getFirstSendTime();
+ if (totalElapsedTime >= config.getTotalRetryTimeout()) {
+ pendingMessages.remove(pending.getSequence());
+ // Decrement InFlight count since we're giving up on this message
+ inFlightCount.decrementAndGet();
+ cancelByLocal(TriRpcStatus.DEADLINE_EXCEEDED.withDescription(
+ "Total retry timeout exceeded for seq: " + pending.getSequence()));
+ return;
+ }
+
+ // 计算指数退避延迟
+ pending.incrementRetry();
+ pending.calculateExponentialBackoff(config);
+
+ // 使用统一的时间管理进行重试,避免重复排程
+ retryMessageWithUnifiedTiming(pending);
+ incrementTotalRetryCount();
+ notifyRetry(pending.getSequence());
+ }
+
+ /**
+ * Retry message with unified timing management.
+ * This updates the sent time for next retry check instead of scheduling independently.
+ */
+ private void retryMessageWithUnifiedTiming(PendingMessage pending) {
+ // Update the sent time to reflect the retry time for unified timing management
+ pending.updateSentTime(System.currentTimeMillis());
+
+ // Execute retry immediately through the unified mechanism
+ retryMessage(pending);
+
+ LOGGER.debug(
+ "Retried message with unified timing for seq: {} (retry count: {})",
+ pending.getSequence(),
+ pending.getRetryCount());
+ }
+
+ private void scheduleRetry(PendingMessage pending) {
+ if (retryScheduler == null || retryScheduler.isShutdown()) {
+ return;
+ }
+ try {
+ ScheduledFuture<?> retryTask = scheduleTrackedTask(
+ retryScheduler,
+ () -> {
+ retryMessage(pending);
+ },
+ config.getRetryTimeout(),
+ TimeUnit.MILLISECONDS);
+
+ if (retryTask != null) {
+ LOGGER.debug("Scheduled retry for message seq: {} for session: {}", pending.getSequence(), sessionId);
+ }
+ } catch (java.util.concurrent.RejectedExecutionException e) {
+ LOGGER.debug(
+ "Failed to schedule retry for seq: {} - scheduler shut down for session: {}",
+ pending.getSequence(),
+ sessionId);
+ }
+ }
+
+ /**
+ * 使用指数退避策略调度重试
+ */
+ private void scheduleRetryWithExponentialBackoff(PendingMessage pending) {
+ if (retryScheduler == null || retryScheduler.isShutdown()) {
+ return;
+ }
+ try {
+ long retryDelay = pending.getNextRetryDelay();
+ LOGGER.debug(
+ "Scheduling exponential backoff retry for seq: {} with delay: {}ms (retry count: {}) for session: {}",
+ pending.getSequence(),
+ retryDelay,
+ pending.getRetryCount(),
+ sessionId);
+
+ ScheduledFuture<?> retryTask = scheduleTrackedTask(
+ retryScheduler,
+ () -> {
+ retryMessage(pending);
+ },
+ retryDelay,
+ TimeUnit.MILLISECONDS);
+
+ if (retryTask != null) {
+ LOGGER.debug(
+ "Scheduled exponential backoff retry task for seq: {} for session: {}",
+ pending.getSequence(),
+ sessionId);
+ }
+ } catch (java.util.concurrent.RejectedExecutionException e) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to schedule exponential backoff retry for seq: " + pending.getSequence() + " in session: "
+ + sessionId,
+ e);
+ }
+ }
+
+ private void retryMessage(PendingMessage pending) {
+ io.netty.handler.codec.http2.DefaultHttp2Headers retryHeaders =
+ new io.netty.handler.codec.http2.DefaultHttp2Headers();
+ retryHeaders.set("tri-seq", String.valueOf(pending.getSequence()));
+ retryHeaders.set("tri-retry", String.valueOf(pending.getRetryCount()));
+
+ HeaderQueueCommand headerCmd =
+ HeaderQueueCommand.createHeaders(getCurrentStreamChannelFuture(), retryHeaders, false);
+ writeQueue.enqueueFuture(headerCmd, parent.eventLoop()).addListener(future -> {
+ if (future.isSuccess()) {
+ DataQueueCommand dataCmd = DataQueueCommand.create(
+ getCurrentStreamChannelFuture(), pending.getMessage(), false, pending.getCompressFlag());
+ writeQueue.enqueueFuture(dataCmd, parent.eventLoop()).addListener(dataResult -> {
+ if (!dataResult.isSuccess()) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Retry data send failed for seq: " + pending.getSequence(),
+ dataResult.cause());
+ }
+ });
+ } else {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Retry header send failed for seq: " + pending.getSequence(),
+ future.cause());
+ }
+ });
+ }
+
+ // getLastAckedSeq() is now implemented in ReliabilityContext interface section
+
+ /**
+ * Remove messages up to the specified sequence number using watermark-based O(1) cleanup.
+ * This is more efficient than scanning the entire map.
+ */
+ private void removeMessagesUpTo(long seq) {
+ // For efficiency, we can use the watermark to quickly identify ranges to remove
+ // Since sequence numbers are monotonically increasing, we can remove all entries <= seq
+ // Use removeIf for safer concurrent map operations
+ if (pendingMessages.isEmpty()) {
+ return;
+ }
+
+ // Use removeIf for thread-safe batch removal
+ int initialSize = pendingMessages.size();
+ boolean removed = pendingMessages.keySet().removeIf(key -> key <= seq);
+
+ if (removed) {
+ int removedCount = initialSize - pendingMessages.size();
+ LOGGER.debug("Removed {} acknowledged messages up to seq: {}", removedCount, seq);
+ }
+ }
+
+ /**
+ * Persist session state (lastAckedSeq, currentSeq) for process restart recovery.
+ * This is called after ACK processing to ensure critical state is persisted.
+ */
+ private void persistSessionState() {
+ try {
+ if (messageStore == null || currentMetadata == null) {
+ return;
+ }
+
+ long currentLastAcked = lastAckedSeq.get();
+ long currentSeq = sequenceNumber.get();
+
+ // Create updated metadata with current state
+ org.apache.dubbo.rpc.protocol.tri.store.SessionMetadata updatedMetadata =
+ currentMetadata.withUpdatedState(currentLastAcked, currentSeq);
+
+ // Persist to storage
+ messageStore.updateMetadata(updatedMetadata);
+
+ // Update reference for next update
+ currentMetadata = updatedMetadata;
+
+ LOGGER.debug(
+ "Persisted session state for session: {} (lastAcked={}, currentSeq={})",
+ sessionId,
+ currentLastAcked,
+ currentSeq);
+
+ } catch (Exception e) {
+ // Don't fail ACK processing if metadata update fails
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Failed to persist session state for session: " + sessionId, e);
+ }
+ }
+
+ private volatile boolean reliabilityResourcesCleaned = false;
+ private final AtomicBoolean reliabilityInitialized = new AtomicBoolean(false);
+ private volatile boolean serverCapabilityNegotiated = false;
+
+ // Failure tracking for intelligent recovery
+ private final AtomicInteger temporaryFailureCount = new AtomicInteger(0);
+ private static final int MAX_TEMPORARY_FAILURES = 5; // Max failures before permanent cleanup
+ private volatile long lastTemporaryFailureTime = 0;
+
+ /**
+ * Determine if an error is recoverable (temporary network issue) or not (protocol error, user cancellation).
+ * Recoverable errors should trigger retry/reconnect, while non-recoverable errors should terminate the stream.
+ *
+ * @param status the error status
+ * @return true if the error is recoverable, false otherwise
+ */
+ private boolean isRecoverableError(TriRpcStatus status) {
+ if (status == null) {
+ return true; // Unknown error, assume recoverable
+ }
+
+ // Non-recoverable errors that should terminate the stream
+ TriRpcStatus.Code code = status.code;
+ switch (code) {
+ case CANCELLED: // User cancelled
+ case INVALID_ARGUMENT: // Bad request
+ case NOT_FOUND: // Service not found
+ case ALREADY_EXISTS: // Duplicate operation
+ case PERMISSION_DENIED: // Authorization error
+ case FAILED_PRECONDITION: // Precondition not met
+ case ABORTED: // Operation aborted
+ case UNIMPLEMENTED: // Method not implemented
+ case UNAUTHENTICATED: // Authentication failed
+ return false; // Don't retry these
+
+ case UNAVAILABLE: // Network/service unavailable - recoverable
+ case DEADLINE_EXCEEDED: // Timeout - may recover
+ case RESOURCE_EXHAUSTED: // Backpressure - may recover
+ case INTERNAL: // Internal error - may be transient
+ case UNKNOWN: // Unknown - assume transient
+ default:
+ return true; // Recoverable, trigger retry
+ }
+ }
+
+ /**
+ * Determine if an exception is recoverable.
+ *
+ * @param throwable the exception
+ * @return true if recoverable, false otherwise
+ */
+ private boolean isRecoverableException(Throwable throwable) {
+ if (throwable == null) {
+ return true;
+ }
+
+ // Network-related exceptions are recoverable
+ String className = throwable.getClass().getName();
+ if (className.contains("IOException")
+ || className.contains("SocketException")
+ || className.contains("ChannelException")
+ || className.contains("ConnectException")
+ || className.contains("TimeoutException")) {
+ return true;
+ }
+
+ // Protocol errors are not recoverable
+ if (className.contains("ProtocolException")
+ || className.contains("CodecException")
+ || className.contains("SerializationException")) {
+ return false;
+ }
+
+ // Default: assume recoverable
+ return true;
+ }
+
+ /**
+ * Attempt recovery for temporary failures, only cleanup after max failures exceeded.
+ * This prevents premature resource cleanup for recoverable issues.
+ */
+ private void attemptRecoveryOrCleanup(String failureReason) {
+ long currentTime = System.currentTimeMillis();
+ int currentFailures = temporaryFailureCount.incrementAndGet();
+ lastTemporaryFailureTime = currentTime;
+
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Temporary failure detected for session: " + sessionId + " - Reason: " + failureReason
+ + ", Failure count: " + currentFailures);
+
+ if (currentFailures >= MAX_TEMPORARY_FAILURES) {
+ LOGGER.error(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Max temporary failures (" + MAX_TEMPORARY_FAILURES + ") exceeded for session: " + sessionId
+ + ", marking as failed but keeping recovery capability");
+ handleTemporaryFailure("Max temporary failures exceeded");
+ } else {
+ // Try to recover instead of immediate cleanup
+ LOGGER.info(
+ "Attempting recovery for session: {} (failure {}/{})",
+ sessionId,
+ currentFailures,
+ MAX_TEMPORARY_FAILURES);
+
+ try {
+ // If we're not in a failed state, try to trigger recovery
+ if (heartbeatState != HeartbeatState.FAILED && heartbeatState != HeartbeatState.CLOSED) {
+ // Schedule recovery attempt with exponential backoff
+ long delayMs = Math.min(1000 * (1L << (currentFailures - 1)), 10000); // Max 10s delay
+
+ if (recoveryExecutor != null && !recoveryExecutor.isShutdown()) {
+ recoveryExecutor.schedule(
+ () -> {
+ try {
+ triggerRecovery();
+ } catch (Exception e) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Recovery attempt failed for session: " + sessionId,
+ e);
+ }
+ },
+ delayMs,
+ java.util.concurrent.TimeUnit.MILLISECONDS);
+ }
+ } else {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Session " + sessionId + " is in terminal state " + heartbeatState
+ + ", cannot attempt recovery");
+ }
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Error during recovery attempt for session: " + sessionId, e);
+ }
+ }
+ }
+
+ /**
+ * Reset temporary failure counter when recovery is successful.
+ */
+ private void resetTemporaryFailureCount() {
+ int previousCount = temporaryFailureCount.getAndSet(0);
+ if (previousCount > 0) {
+ LOGGER.info("Reset temporary failure count for session: {} (was: {})", sessionId, previousCount);
+ }
+ }
+
+ /**
+ * Atomically switch stream channel future to ensure thread safety during reconnection.
+ */
+ private boolean switchStreamChannelAtomically(TripleStreamChannelFuture newChannelFuture) {
+ TripleStreamChannelFuture oldChannelFuture = streamChannelFuture.getAndSet(newChannelFuture);
+
+ if (oldChannelFuture != null) {
+ // Close the old channel to prevent resource leaks
+ try {
+ Object oldChannel = oldChannelFuture.getNow();
+ if (oldChannel instanceof Channel && ((Channel) oldChannel).isActive()) {
+ ((Channel) oldChannel).close();
+ LOGGER.debug("Closed old stream channel during atomic switch for session: {}", sessionId);
+ }
+ } catch (Exception e) {
+ LOGGER.debug("Error closing old stream channel during atomic switch for session: {}", sessionId, e);
+ }
+ }
+
+ LOGGER.info("Atomically switched stream channel future for session: {}", sessionId);
+ return true;
+ }
+
+ /**
+ * Get current stream channel future safely.
+ */
+ private TripleStreamChannelFuture getCurrentStreamChannelFuture() {
+ return streamChannelFuture.get();
+ }
+
+ /**
+ * Handle temporary failures that should not permanently disable reliability.
+ * This method allows recovery after temporary network issues or connection problems.
+ * This version increments the failure count and should be used for independent failure events.
+ */
+ private void handleTemporaryFailureWithCount(String reason) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Handling temporary failure for session " + sessionId + ": " + reason);
+
+ // Transition to FAILED state to indicate current connection issues
+ heartbeatState = HeartbeatState.FAILED;
+
+ // Cancel current heartbeat/retry tasks but don't shutdown executors
+ cancelCurrentScheduledTasks();
+
+ // Clear current stream channel future to force new connection
+ streamChannelFuture.set(null);
+
+ // Increment failure count for tracking
+ int currentFailureCount = temporaryFailureCount.incrementAndGet();
+
+ // Do NOT set reliabilityEnabled = false - keep it enabled for recovery
+ // Do NOT clear pending messages - they will be resent after recovery
+
+ LOGGER.info(
+ "Temporary failure handled for session {}, failure count: {}, state: {}",
+ sessionId,
+ currentFailureCount,
+ heartbeatState);
+ }
+
+ /**
+ * Handle temporary failures without incrementing count.
+ * This method should be used when the failure count has already been incremented by the caller.
+ */
+ private void handleTemporaryFailure(String reason) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Handling temporary failure for session " + sessionId + ": " + reason);
+
+ // Transition to FAILED state to indicate current connection issues
+ heartbeatState = HeartbeatState.FAILED;
+
+ // Cancel current heartbeat/retry tasks but don't shutdown executors
+ cancelCurrentScheduledTasks();
+
+ // Clear current stream channel future to force new connection
+ streamChannelFuture.set(null);
+
+ // Do NOT increment failure count here - it should be done by the caller
+ // Do NOT set reliabilityEnabled = false - keep it enabled for recovery
+ // Do NOT clear pending messages - they will be resent after recovery
+
+ int currentFailureCount = temporaryFailureCount.get();
+ LOGGER.info(
+ "Temporary failure handled for session {}, failure count: {}, state: {}",
+ sessionId,
+ currentFailureCount,
+ heartbeatState);
+ }
+
+ /**
+ * Cancel currently scheduled tasks without shutting down executors.
+ * This method now properly cancels individual scheduled tasks to avoid resource waste.
+ */
+ private void cancelCurrentScheduledTasks() {
+ try {
+ int cancelledCount = 0;
+
+ // Cancel current heartbeat task
+ if (currentHeartbeatTask != null && !currentHeartbeatTask.isDone()) {
+ boolean cancelled = currentHeartbeatTask.cancel(false);
+ if (cancelled) {
+ cancelledCount++;
+ LOGGER.debug("Cancelled current heartbeat task for session: {}", sessionId);
+ }
+ currentHeartbeatTask = null;
+ }
+
+ // Cancel current retry task
+ if (currentRetryTask != null && !currentRetryTask.isDone()) {
+ boolean cancelled = currentRetryTask.cancel(false);
+ if (cancelled) {
+ cancelledCount++;
+ LOGGER.debug("Cancelled current retry task for session: {}", sessionId);
+ }
+ currentRetryTask = null;
+ }
+
+ // Cancel all other active tasks
+ for (ScheduledFuture<?> task : activeTasks) {
+ if (task != null && !task.isDone()) {
+ boolean cancelled = task.cancel(false);
+ if (cancelled) {
+ cancelledCount++;
+ }
+ }
+ }
+ activeTasks.clear();
+
+ LOGGER.debug("Cancelled {} scheduled tasks for session: {}", cancelledCount, sessionId);
+
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error during task cancellation for session: " + sessionId, e);
+ }
+ }
+
+ /**
+ * Schedule a task and track it for proper cancellation.
+ */
+ private ScheduledFuture<?> scheduleTrackedTask(
+ ScheduledExecutorService executor, Runnable task, long delay, TimeUnit unit) {
+ try {
+ // Create wrapper class for self-tracking
+ class TrackedTask implements Runnable {
+ private ScheduledFuture<?> taskFuture;
+
+ @Override
+ public void run() {
+ try {
+ task.run();
+ } finally {
+ if (taskFuture != null) {
+ activeTasks.remove(taskFuture);
+ }
+ }
+ }
+
+ public void setFuture(ScheduledFuture<?> future) {
+ this.taskFuture = future;
+ }
+ }
+
+ TrackedTask wrappedTask = new TrackedTask();
+ ScheduledFuture<?> future = executor.schedule(wrappedTask, delay, unit);
+ wrappedTask.setFuture(future);
+ activeTasks.add(future);
+
+ return future;
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error scheduling tracked task for session: " + sessionId, e);
+ return null;
+ }
+ }
+
+ private ScheduledFuture<?> scheduleTrackedTaskAtFixedRate(
+ ScheduledExecutorService executor, Runnable task, long initialDelay, long period, TimeUnit unit) {
+ try {
+ ScheduledFuture<?> future = executor.scheduleAtFixedRate(task, initialDelay, period, unit);
+ activeTasks.add(future);
+
+ // Note: Fixed-rate tasks don't self-remove since they run indefinitely
+ // They will be removed only when explicitly cancelled
+ return future;
+ } catch (Exception e) {
+ LOGGER.warn(
+ INTERNAL_ERROR, "", "", "Error scheduling tracked fixed-rate task for session: " + sessionId, e);
+ return null;
+ }
+ }
+
+ private void cleanupReliabilityResources() {
+ // Early return if reliability was never enabled
+ if (!reliabilityEnabled && !reliabilityInitialized.get()) {
+ return;
+ }
+
+ // Prevent multiple cleanup calls using atomic flag
+ if (reliabilityResourcesCleaned) {
+ LOGGER.debug("Reliability resources already cleaned for session: {}", sessionId);
+ return;
+ }
+
+ // Use synchronized block to ensure thread-safe cleanup
+ synchronized (this) {
+ if (reliabilityResourcesCleaned) {
+ return;
+ }
+ reliabilityResourcesCleaned = true;
+ }
+
+ try {
+ LOGGER.info("Starting reliability resources cleanup for session: {}", sessionId);
+
+ // Reset initialization flag to allow future initialization
+ reliabilityInitialized.set(false);
+
+ // Cancel any pending scheduled heartbeat tasks first
+ cancelAllScheduledTasks();
+
+ // Shutdown heartbeat scheduler with proper timeout and error handling
+ shutdownExecutorSafely(heartbeatScheduler, "heartbeat scheduler", 500);
+
+ // Shutdown retry scheduler with proper timeout and error handling
+ shutdownExecutorSafely(retryScheduler, "retry scheduler", 500);
+
+ // Shutdown recovery executor with proper timeout and error handling
+ if (recoveryExecutor != null) {
+ shutdownExecutorSafely(recoveryExecutor, "recovery executor", 500);
+ }
+
+ // Clear all pending messages and notify them of cancellation
+ clearPendingMessagesWithNotification();
+
+ // Close message store
+ if (messageStore != null) {
+ try {
+ SessionMetadata metadata = new SessionMetadata(
+ sessionId,
+ url != null ? url.getServiceKey() : "unknown",
+ System.currentTimeMillis(),
+ url,
+ config);
+ messageStore.close(metadata);
+ LOGGER.debug("Closed message store for session: {}", sessionId);
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error closing message store for session: " + sessionId, e);
+ }
+ }
+
+ // Reset reliability state to prevent further operations
+ resetReliabilityState();
+
+ LOGGER.debug("Successfully cleaned up all reliability resources for session: {}", sessionId);
+ } catch (Exception e) {
+ LOGGER.warn(
+ INTERNAL_ERROR, "", "", "Error during reliability resource cleanup for session: " + sessionId, e);
+ }
+ }
+
+ /**
+ * Pause reliability temporarily while preserving recovery capability.
+ * Unlike cleanup, this maintains messageStore and pending messages for later recovery.
+ */
+ private void pauseReliability() {
+ try {
+ LOGGER.info("Pausing reliability for session: {} (preserving recovery capability)", sessionId);
+
+ // Cancel any pending scheduled tasks but don't shutdown executors completely
+ cancelAllScheduledTasks();
+
+ // Transition to PAUSED state to indicate temporary suspension
+ transitionToState(HeartbeatState.PAUSED, "Temporarily paused due to connection issues");
+
+ // Keep reliabilityEnabled = true and preserve messageStore and pendingMessages
+ // This allows for later recovery without losing state
+
+ LOGGER.info("Successfully paused reliability for session: {} (can be recovered)", sessionId);
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error during reliability pause for session: " + sessionId, e);
+ }
+ }
+
+ /**
+ * Reset reliability state to paused instead of completely disabled.
+ * This preserves the ability to recover later.
+ */
+ private void pauseReliabilityState() {
+ try {
+ // Do NOT reset counters and sequence numbers - preserve them for recovery
+ // Do NOT set reliabilityEnabled = false - keep it true for recovery
+
+ // Only reset heartbeat timing
+ missedHeartbeats = 0;
+ lastHeartbeatSentTime = 0;
+ lastHeartbeatAckTime = 0;
+
+ // Transition to paused state instead of closed
+ if (heartbeatState != null && heartbeatState != HeartbeatState.CLOSED) {
+ heartbeatState = HeartbeatState.PAUSED;
+ }
+
+ LOGGER.debug("Paused reliability state for session: {} (preserving recovery capability)", sessionId);
+ } catch (Exception e) {
+ LOGGER.debug("Error pausing reliability state", e);
+ }
}
- @Override
- public ChannelFuture sendMessage(byte[] message, int compressFlag) {
- ChannelFuture checkResult = preCheck();
- if (!checkResult.isSuccess()) {
- return checkResult;
+ private void shutdownExecutorSafely(java.util.concurrent.ExecutorService executor, String name, long timeoutMs) {
+ if (executor == null || executor.isShutdown()) {
+ return;
}
- final DataQueueCommand cmd = DataQueueCommand.create(streamChannelFuture, message, false, compressFlag);
- return this.writeQueue.enqueueFuture(cmd, parent.eventLoop()).addListener(future -> {
- if (!future.isSuccess()) {
- cancelByLocal(TriRpcStatus.INTERNAL
- .withDescription("Client write message failed")
- .withCause(future.cause()));
- transportException(future.cause());
+
+ try {
+ // First, try graceful shutdown
+ executor.shutdown();
+ if (!executor.awaitTermination(timeoutMs / 2, java.util.concurrent.TimeUnit.MILLISECONDS)) {
+ // If graceful shutdown fails, force shutdown
+ LOGGER.debug("Graceful shutdown of {} timed out, forcing shutdown", name);
+ executor.shutdownNow();
+ // Wait for forced shutdown to complete
+ if (!executor.awaitTermination(timeoutMs / 2, java.util.concurrent.TimeUnit.MILLISECONDS)) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Failed to shutdown " + name + " within " + timeoutMs + "ms");
+ }
}
- });
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOGGER.debug("Interrupted while shutting down {}, forcing immediate shutdown", name);
+ executor.shutdownNow();
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error shutting down " + name, e);
+ try {
+ executor.shutdownNow();
+ } catch (Exception ex) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error during forced shutdown of " + name, ex);
+ }
+ }
}
- @Override
- public void request(int n) {
- deframer.request(n);
+ private void cancelAllScheduledTasks() {
+ // Cancel scheduled tasks by shutting down gracefully first
+ try {
+ if (heartbeatScheduler != null && !heartbeatScheduler.isShutdown()) {
+ // For ScheduledExecutorService, we shutdown gracefully which cancels pending tasks
+ LOGGER.debug("Cancelling heartbeat scheduled tasks for session: {}", sessionId);
+ }
+ if (retryScheduler != null && !retryScheduler.isShutdown()) {
+ // Cancel retry scheduled tasks
+ LOGGER.debug("Cancelling retry scheduled tasks for session: {}", sessionId);
+ }
+ } catch (Exception e) {
+ LOGGER.debug("Error cancelling scheduled tasks", e);
+ }
}
- @Override
- public ChannelFuture halfClose() {
- ChannelFuture checkResult = preCheck();
- if (!checkResult.isSuccess()) {
- return checkResult;
+ private void clearPendingMessagesWithNotification() {
+ if (pendingMessages == null || pendingMessages.isEmpty()) {
+ return;
}
- final EndStreamQueueCommand cmd = EndStreamQueueCommand.create(streamChannelFuture);
- return this.writeQueue.enqueueFuture(cmd, parent.eventLoop()).addListener(future -> {
- if (future.isSuccess()) {
- halfClosed = true;
+
+ try {
+ // Notify all pending messages of cancellation before clearing
+ TriRpcStatus cancellationStatus = TriRpcStatus.CANCELLED.withDescription("Stream closed");
+ pendingMessages.values().forEach(pending -> {
+ try {
+ // Log pending message being cancelled
+ LOGGER.debug(
+ "Cancelling pending message seq: {} for session: {}", pending.getSequence(), sessionId);
+ } catch (Exception e) {
+ LOGGER.debug("Error notifying pending message cancellation", e);
+ }
+ });
+
+ // Clear the map
+ pendingMessages.clear();
+ LOGGER.debug("Cleared {} pending messages for session: {}", pendingMessages.size(), sessionId);
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error clearing pending messages", e);
+ // Force clear even if notification fails
+ try {
+ pendingMessages.clear();
+ } catch (Exception ex) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error force-clearing pending messages", ex);
}
- });
+ }
}
- private ChannelFuture preCheck() {
- if (rst) {
- return streamChannelFuture.getNow().newFailedFuture(new IOException("stream channel has reset"));
+ private void resetReliabilityState() {
+ try {
+ // Reset counters and state
+ lastAckedSeq.set(0);
+ sequenceNumber.set(0);
+ missedHeartbeats = 0;
+ lastHeartbeatSentTime = 0;
+ lastHeartbeatAckTime = 0;
+ reliabilityEnabled = false;
+
+ // Transition to closed state
+ if (heartbeatState != null) {
+ heartbeatState = HeartbeatState.CLOSED;
+ }
+ } catch (Exception e) {
+ LOGGER.debug("Error resetting reliability state", e);
}
- return parent.newSucceededFuture();
}
/**
* @return transport listener
*/
- protected H2TransportListener createTransportListener() {
+ public H2TransportListener createTransportListener() {
return new ClientTransportListener();
}
+ /**
+ * Get the current transport listener for reuse during reconnection.
+ * This ensures we maintain the same listener instance across reconnection cycles.
+ *
+ * @return current transport listener
+ */
+ public H2TransportListener getTransportListener() {
+ return this.transportListener;
+ }
+
class ClientTransportListener extends AbstractH2TransportListener implements H2TransportListener {
private TriRpcStatus transportError;
@@ -215,12 +2883,19 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
private Http2Headers trailers;
void handleH2TransportError(TriRpcStatus status) {
- writeQueue.enqueue(CancelQueueCommand.createCommand(streamChannelFuture, Http2Error.NO_ERROR));
+ writeQueue.enqueue(CancelQueueCommand.createCommand(getCurrentStreamChannelFuture(), Http2Error.NO_ERROR));
rst = true;
finishProcess(status, null, false);
}
void finishProcess(TriRpcStatus status, Http2Headers trailers, boolean isReturnTriException) {
+ try {
+ // Clean up reliability resources when finishing
+ cleanupReliabilityResources();
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error during reliability cleanup in finishProcess", e);
+ }
+
final Map<CharSequence, String> reserved = filterReservedHeaders(trailers);
final Map<String, Object> attachments =
headersToMap(trailers, () -> reserved.get(TripleHeaderEnum.TRI_HEADER_CONVERT.getKey()));
@@ -258,6 +2933,11 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
transportError = TriRpcStatus.INTERNAL.withDescription("Received headers twice");
return;
}
+
+ // Process reliability headers first
+ if (reliabilityEnabled) {
+ processReliabilityHeaders(headers);
+ }
Integer httpStatus = headers.status() == null
? null
: Integer.parseInt(headers.status().toString());
@@ -412,21 +3092,27 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("endStream: {} HEADERS: {}", endStream, headers);
}
- executor.execute(() -> {
- if (endStream) {
- if (!halfClosed) {
- Channel channel = streamChannelFuture.getNow();
- if (channel.isActive() && !rst) {
- writeQueue.enqueue(
- CancelQueueCommand.createCommand(streamChannelFuture, Http2Error.CANCEL));
- rst = true;
+ try {
+ executor.execute(() -> {
+ if (endStream) {
+ if (!halfClosed) {
+ TripleStreamChannelFuture currentFuture = getCurrentStreamChannelFuture();
+ Channel channel = currentFuture != null ? currentFuture.getNow() : null;
+ if (channel.isActive() && !rst) {
+ writeQueue.enqueue(CancelQueueCommand.createCommand(
+ getCurrentStreamChannelFuture(), Http2Error.CANCEL));
+ rst = true;
+ }
}
+ onTrailersReceived(headers);
+ } else {
+ onHeaderReceived(headers);
}
- onTrailersReceived(headers);
- } else {
- onHeaderReceived(headers);
- }
- });
+ });
+ } catch (Throwable t) {
+ LOGGER.error(PROTOCOL_FAILED_RESPONSE, "", "", "submit onHeader task failed", t);
+ AbstractTripleClientStream.this.attemptRecoveryOrCleanup("onHeader task submission failed");
+ }
}
@Override
@@ -442,6 +3128,7 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
// For example, ThreadLessExecutor will be shutdown when request timeout {@link AsyncRpcResult}
ReferenceCountUtil.release(data);
LOGGER.error(PROTOCOL_FAILED_RESPONSE, "", "", "submit onData task failed", t);
+ AbstractTripleClientStream.this.attemptRecoveryOrCleanup("onData task submission failed");
}
}
@@ -463,16 +3150,500 @@ public abstract class AbstractTripleClientStream extends AbstractStream implemen
@Override
public void cancelByRemote(long errorCode) {
- executor.execute(() -> {
- transportError =
- TriRpcStatus.CANCELLED.withDescription("Canceled by remote peer, errorCode=" + errorCode);
- finishProcess(transportError, null, false);
- });
+ try {
+ executor.execute(() -> {
+ transportError =
+ TriRpcStatus.CANCELLED.withDescription("Canceled by remote peer, errorCode=" + errorCode);
+ finishProcess(transportError, null, false);
+ });
+ } catch (Throwable t) {
+ LOGGER.error(PROTOCOL_FAILED_RESPONSE, "", "", "submit cancelByRemote task failed", t);
+ AbstractTripleClientStream.this.attemptRecoveryOrCleanup("cancelByRemote task submission failed");
+ }
+ }
+
+ private void processReliabilityHeaders(Http2Headers headers) {
+ // First check server capability negotiation on first response
+ if (!serverCapabilityNegotiated) {
+ checkServerCapabilityNegotiation(headers);
+ serverCapabilityNegotiated = true;
+ }
+
+ // Enhanced ACK header processing
+ CharSequence ackHeader = headers.get("tri-ack");
+ if (ackHeader != null) {
+ // Debug log to confirm ACK headers were received
+ LOGGER.debug(
+ "Client received ACK headers: tri-ack={}, tri-last-acked={}, session={}",
+ ackHeader,
+ headers.get("tri-last-acked"),
+ sessionId);
+ long ackedSeq = Long.parseLong(ackHeader.toString());
+ // Update the last acknowledged sequence number
+ long previousAckedSeq = lastAckedSeq.getAndSet(ackedSeq);
+
+ // Calculate how many messages were acknowledged in this batch
+ long newlyAckedCount = ackedSeq - previousAckedSeq;
+ if (newlyAckedCount > 0) {
+ // Atomic decrement with boundary protection using compareAndSet loop
+ int current, updated;
+ do {
+ current = inFlightCount.get();
+ updated = Math.max(0, current - (int) newlyAckedCount);
+ } while (!inFlightCount.compareAndSet(current, updated));
+
+ if (current < newlyAckedCount) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "InFlight count protection triggered: attempted to ack " + newlyAckedCount
+ + " messages but only " + current + " were in flight. "
+ + "This may indicate duplicate ACKs or sequence number issues. Corrected to: "
+ + updated);
+ }
+
+ LOGGER.debug(
+ "Acknowledged {} messages, InFlight count: {} -> {}", newlyAckedCount, current, updated);
+ }
+
+ // 持久化存储ACK
+ if (messageStore != null && messageStore.isHealthy()) {
+ try {
+ messageStore.ack(ackedSeq);
+ } catch (StoreException e) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Failed to persist ack for sequence: " + ackedSeq + " for session: " + sessionId,
+ e);
+ }
+ }
+
+ // Remove acknowledged messages using watermark-based O(1) cleanup
+ long oldWatermark = ackWatermark.get();
+ if (ackedSeq > oldWatermark) {
+ // Update watermark and clean up in batch
+ ackWatermark.compareAndSet(oldWatermark, ackedSeq);
+ removeMessagesUpTo(ackedSeq);
+ }
+
+ // Persist updated session state for process restart recovery
+ persistSessionState();
+
+ return;
+ }
+
+ CharSequence nakHeader = headers.get("tri-nak");
+ if (nakHeader != null) {
+ long nakedSeq = Long.parseLong(nakHeader.toString());
+ PendingMessage pending = pendingMessages.get(nakedSeq);
+ if (pending != null) {
+ // Use unified timing management for NAK-triggered retries as well
+ checkAndRetryWithUnifiedTiming(pending, System.currentTimeMillis());
+ }
+ return;
+ }
+
+ CharSequence heartbeatAck = headers.get("tri-heartbeat-ack");
+ if (heartbeatAck != null) {
+ onHeartbeatAck();
+ return;
+ }
+
+ // Check for heartbeat from server (for bidirectional heartbeat)
+ CharSequence serverHeartbeat = headers.get("tri-heartbeat");
+ if (serverHeartbeat != null) {
+ sendHeartbeatAckToServer(serverHeartbeat.toString());
+ }
+ }
+
+ /**
+ * Check server capability negotiation in first response.
+ * If server doesn't support reliability, fallback to non-reliable mode.
+ */
+ private void checkServerCapabilityNegotiation(Http2Headers headers) {
+ CharSequence serverAck = headers.get("tri-reliable-ack");
+
+ if (serverAck == null) {
+ // Server doesn't support reliability, fallback to non-reliable mode
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Server doesn't support reliability (no tri-reliable-ack header), falling back to non-reliable mode for session: "
+ + sessionId);
+
+ // Disable reliability and cleanup resources
+ performReliabilityFallback("Server capability negotiation failed");
+
+ } else {
+ LOGGER.info(
+ "Server confirmed reliability support (tri-reliable-ack: {}) for session: {}",
+ serverAck,
+ sessionId);
+ // Server supports reliability, continue with reliable mode
+ }
+ }
+
+ /**
+ * Perform fallback to non-reliable mode due to server incompatibility.
+ */
+ private void performReliabilityFallback(String reason) {
+ try {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Performing reliability fallback for session: " + sessionId + " - Reason: " + reason);
+
+ // Transition to CLOSED state to prevent further reliability operations
+ transitionToState(HeartbeatState.CLOSED, reason);
+
+ // Disable reliability permanently since server doesn't support it
+ reliabilityEnabled = false;
+
+ // Clean up reliability resources since server is incompatible
+ cleanupReliabilityResources();
+
+ // Clear pending messages since they won't be ACKed
+ if (pendingMessages != null) {
+ int clearedCount = pendingMessages.size();
+ pendingMessages.clear();
+ LOGGER.info(
+ "Cleared {} pending messages during reliability fallback for session: {}",
+ clearedCount,
+ sessionId);
+ }
+
+ // Reset counters
+ if (inFlightCount != null) {
+ inFlightCount.set(0);
+ }
+
+ LOGGER.info("Successfully completed reliability fallback for session: {}", sessionId);
+
+ } catch (Exception e) {
+ LOGGER.error(INTERNAL_ERROR, "", "", "Error during reliability fallback for session: " + sessionId, e);
+ }
}
@Override
public void onClose() {
- executor.execute(listener::onClose);
+ // Clean up reliability resources when closing
+ cleanupReliabilityResources();
+ try {
+ executor.execute(listener::onClose);
+ } catch (Throwable t) {
+ LOGGER.error(PROTOCOL_FAILED_RESPONSE, "", "", "submit onClose task failed", t);
+ // Note: cleanup() not called here as this is already the cleanup path
+ }
+ }
+ }
+
+ // PendingMessage class for reliability tracking
+ private static class PendingMessage {
+ private final long sequence;
+ private final byte[] message;
+ private final int compressFlag;
+ private volatile long sentTime;
+ private volatile int retryCount;
+ private volatile long firstSendTime; // 首次发送时间,用于总重试超时控制
+ private volatile long nextRetryDelay; // 下次重试延迟时间 (ms)
+
+ public PendingMessage(long sequence, byte[] message, int compressFlag, long sentTime, int retryCount) {
+ this.sequence = sequence;
+ this.message = message;
+ this.compressFlag = compressFlag;
+ this.sentTime = sentTime;
+ this.retryCount = retryCount;
+ this.firstSendTime = sentTime; // 初始化首次发送时间
+ this.nextRetryDelay = 100; // 初始重试延迟 100ms
}
+
+ public long getSequence() {
+ return sequence;
+ }
+
+ public byte[] getMessage() {
+ return message;
+ }
+
+ public int getCompressFlag() {
+ return compressFlag;
+ }
+
+ public long getSentTime() {
+ return sentTime;
+ }
+
+ public int getRetryCount() {
+ return retryCount;
+ }
+
+ public void incrementRetry() {
+ retryCount++;
+ }
+
+ public void updateSentTime(long newTime) {
+ sentTime = newTime;
+ }
+
+ public long getFirstSendTime() {
+ return firstSendTime;
+ }
+
+ public long getNextRetryDelay() {
+ return nextRetryDelay;
+ }
+
+ public void setNextRetryDelay(long delay) {
+ this.nextRetryDelay = delay;
+ }
+
+ public void calculateExponentialBackoff(ReliabilityConfig config) {
+ // 指数退避计算:delay = min(initialDelay * multiplier^retryCount, maxDelay)
+ long calculatedDelay =
+ (long) (config.getInitialRetryDelay() * Math.pow(config.getBackoffMultiplier(), this.retryCount));
+ this.nextRetryDelay = Math.min(calculatedDelay, config.getMaxRetryDelay());
+ }
+ }
+
+ /**
+ * Check if a message is still retryable based on total timeout and max retry count.
+ * This ensures reconnection-triggered resends also respect retry limits.
+ *
+ * @param pending the pending message to check
+ * @return true if the message can still be retried, false if it should be dropped
+ */
+ private boolean isStillRetryable(PendingMessage pending) {
+ if (pending == null || config == null) {
+ return false;
+ }
+
+ // Check 1: Max retry count
+ if (pending.getRetryCount() >= config.getMaxRetries()) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Message seq=" + pending.getSequence() + " exceeded max retries (" + pending.getRetryCount() + "/"
+ + config.getMaxRetries() + "), dropping");
+ return false;
+ }
+
+ // Check 2: Total retry timeout (time since first send)
+ long elapsedTime = System.currentTimeMillis() - pending.getFirstSendTime();
+ long totalTimeout = config.getTotalRetryTimeout();
+
+ if (totalTimeout > 0 && elapsedTime > totalTimeout) {
+ LOGGER.warn(
+ INTERNAL_ERROR,
+ "",
+ "",
+ "Message seq=" + pending.getSequence() + " exceeded total retry timeout (" + elapsedTime + "ms > "
+ + totalTimeout + "ms), dropping");
+ return false;
+ }
+
+ return true;
+ }
+
+ // ========== ReliabilityContext Implementation ==========
+
+ @Override
+ public String getState() {
+ if (!reliabilityEnabled) {
+ return "DISABLED";
+ }
+ return heartbeatState != null ? heartbeatState.name() : "UNKNOWN";
+ }
+
+ @Override
+ public long getLastAckedSeq() {
+ return reliabilityEnabled ? lastAckedSeq.get() : -1;
+ }
+
+ @Override
+ public int getPendingCount() {
+ return reliabilityEnabled ? pendingMessages.size() : 0;
+ }
+
+ @Override
+ public int getInFlightCount() {
+ return reliabilityEnabled ? inFlightCount.get() : 0;
+ }
+
+ @Override
+ public String getSessionId() {
+ return reliabilityEnabled ? sessionId : null;
+ }
+
+ @Override
+ public long getTotalSentCount() {
+ return reliabilityEnabled ? totalSentCount.get() : 0;
+ }
+
+ @Override
+ public long getTotalRetryCount() {
+ return reliabilityEnabled ? totalRetryCount.get() : 0;
+ }
+
+ @Override
+ public boolean isConnectionActive() {
+ if (!reliabilityEnabled) {
+ return false;
+ }
+ return reconnectionManager != null && reconnectionManager.isConnectionActive();
+ }
+
+ @Override
+ public java.util.Map<String, Object> getStatistics() {
+ java.util.Map<String, Object> stats = new java.util.HashMap<>();
+ if (reliabilityEnabled) {
+ stats.put("state", getState());
+ stats.put("lastAckedSeq", getLastAckedSeq());
+ stats.put("pendingCount", getPendingCount());
+ stats.put("inFlightCount", getInFlightCount());
+ stats.put("sessionId", getSessionId());
+ stats.put("totalSentCount", getTotalSentCount());
+ stats.put("totalRetryCount", getTotalRetryCount());
+ stats.put("connectionActive", isConnectionActive());
+ stats.put("heartbeatState", heartbeatState != null ? heartbeatState.name() : "UNKNOWN");
+ if (config != null) {
+ stats.put("maxRetries", config.getMaxRetries());
+ stats.put("retryTimeout", config.getRetryTimeout());
+ stats.put("maxInFlightMessages", config.getMaxInFlightMessages());
+ }
+ }
+ return stats;
+ }
+
+ @Override
+ public void onStateChange(Consumer<String> callback) {
+ this.stateChangeCallback = callback;
+ }
+
+ @Override
+ public void onRecovery(Consumer<String> callback) {
+ this.recoveryCallback = callback;
+ }
+
+ @Override
+ public void onRetry(Consumer<Long> callback) {
+ this.retryCallback = callback;
+ }
+
+ @Override
+ public boolean retryMessage(long sequence) {
+ if (!reliabilityEnabled) {
+ return false;
+ }
+
+ PendingMessage pending = pendingMessages.get(sequence);
+ if (pending != null) {
+ checkAndRetryWithUnifiedTiming(pending, System.currentTimeMillis());
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean triggerRecovery() {
+ // Prevent concurrent reconnection attempts
+ if (heartbeatState == HeartbeatState.RECONNECTING) {
+ return false;
+ }
+
+ // Allow recovery from FAILED state even if reliability was disabled
+ if (heartbeatState == HeartbeatState.FAILED) {
+ LOGGER.info("Triggering recovery from FAILED state for session: {}", sessionId);
+ return recoverFromFailedState();
+ }
+
+ // Allow recovery from PAUSED state (should preserve reliability settings)
+ if (heartbeatState == HeartbeatState.PAUSED) {
+ LOGGER.info("Triggering recovery from PAUSED state for session: {}", sessionId);
+ return recoverFromPausedState();
+ }
+
+ // For other states, require reliability to be enabled
+ if (!reliabilityEnabled) {
+ return false;
+ }
+
+ attemptRealReconnection();
+ return true;
+ }
+
+ @Override
+ public void setInFlightLimit(int limit) {
+ if (!reliabilityEnabled || config == null) {
+ return;
+ }
+ // Note: This is a simplified implementation. In a production environment,
+ // you would want to create a new config instance or use a builder pattern.
+ LOGGER.info("InFlight limit updated to: {}", limit);
+ }
+
+ @Override
+ public ReliabilityConfig getRetryConfig() {
+ return reliabilityEnabled ? config : null;
+ }
+
+ // ========== Helper methods for ReliabilityContext ==========
+
+ /**
+ * Update reliability metrics and trigger callbacks when state changes.
+ */
+ private void updateReliabilityState(String newState, String reason) {
+ if (stateChangeCallback != null) {
+ try {
+ stateChangeCallback.accept(newState + ": " + reason);
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error in state change callback", e);
+ }
+ }
+ }
+
+ /**
+ * Trigger recovery callback when connection is recovered.
+ */
+ private void notifyRecovery(String reason) {
+ if (recoveryCallback != null) {
+ try {
+ recoveryCallback.accept(reason);
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error in recovery callback", e);
+ }
+ }
+ }
+
+ /**
+ * Trigger retry callback when a message is retried.
+ */
+ private void notifyRetry(long sequence) {
+ if (retryCallback != null) {
+ try {
+ retryCallback.accept(sequence);
+ } catch (Exception e) {
+ LOGGER.warn(INTERNAL_ERROR, "", "", "Error in retry callback", e);
+ }
+ }
+ }
+
+ /**
+ * Increment total sent count and trigger callbacks.
+ */
+ private void incrementTotalSentCount() {
+ totalSentCount.incrementAndGet();
+ }
+
+ /**
+ * Increment total retry count and trigger callbacks.
+ */
+ private void incrementTotalRetryCount() {
+ totalRetryCount.incrementAndGet();
}
}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReconnectionManager.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReconnectionManager.java
new file mode 100644
index 0000000000..a9c507af31
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReconnectionManager.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.stream;
+
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Interface for managing stream reconnection.
+ * Provides callback mechanisms for reliable streaming to handle connection failures.
+ */
+public interface ReconnectionManager {
+
+ /**
+ * Attempt to reconnect the underlying connection.
+ * This should trigger a new connection establishment and return a new stream channel.
+ *
+ * @return CompletableFuture that completes with true if reconnection succeeded, false otherwise
+ */
+ CompletableFuture<Boolean> attemptReconnection();
+
+ /**
+ * Check if the underlying connection is currently active.
+ *
+ * @return true if connection is active, false otherwise
+ */
+ boolean isConnectionActive();
+
+ /**
+ * Get a new stream channel after successful reconnection.
+ * This should only be called after successful reconnection.
+ *
+ * @return new stream channel, or null if no connection available
+ * @deprecated Use {@link #createStreamFuture()} for type-safe channel creation
+ */
+ @Deprecated
+ Object getNewStreamChannel();
+
+ /**
+ * Create a new TripleStreamChannelFuture after successful reconnection.
+ * This provides type-safe channel creation for reliable streaming.
+ *
+ * @return new TripleStreamChannelFuture, or null if no connection available
+ */
+ TripleStreamChannelFuture createStreamFuture();
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReliabilityConfig.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReliabilityConfig.java
new file mode 100644
index 0000000000..e4256e105d
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReliabilityConfig.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.stream;
+
+public class ReliabilityConfig {
+ private final boolean enabled;
+ private final int heartbeatInterval;
+ private final int retryTimeout;
+ private final int maxRetries;
+ private final int sessionTimeout;
+ private final int maxMissedHeartbeats;
+
+ // 智能重试策略参数
+ private final int initialRetryDelay; // 初始重试延迟 (ms)
+ private final int maxRetryDelay; // 最大重试延迟 (ms)
+ private final double backoffMultiplier; // 退避倍数
+ private final int maxInFlightMessages; // 最大 InFlight 消息数
+ private final int totalRetryTimeout; // 总重试超时时间 (ms)
+
+ // 持久化存储配置
+ private final String storeType; // 存储类型: memory, file, redis
+ private final String storePath; // 存储路径
+ private final long storeMaxFileSize; // 存储文件最大大小 (bytes)
+ private final long storeRetentionTime; // 存储保留时间 (ms)
+ private final int storeSyncInterval; // 存储同步间隔 (ms)
+
+ private ReliabilityConfig(Builder builder) {
+ this.enabled = builder.enabled;
+ this.heartbeatInterval = builder.heartbeatInterval;
+ this.retryTimeout = builder.retryTimeout;
+ this.maxRetries = builder.maxRetries;
+ this.sessionTimeout = builder.sessionTimeout;
+ this.maxMissedHeartbeats = builder.maxMissedHeartbeats;
+ this.initialRetryDelay = builder.initialRetryDelay;
+ this.maxRetryDelay = builder.maxRetryDelay;
+ this.backoffMultiplier = builder.backoffMultiplier;
+ this.maxInFlightMessages = builder.maxInFlightMessages;
+ this.totalRetryTimeout = builder.totalRetryTimeout;
+ this.storeType = builder.storeType;
+ this.storePath = builder.storePath;
+ this.storeMaxFileSize = builder.storeMaxFileSize;
+ this.storeRetentionTime = builder.storeRetentionTime;
+ this.storeSyncInterval = builder.storeSyncInterval;
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public int getHeartbeatInterval() {
+ return heartbeatInterval;
+ }
+
+ public int getRetryTimeout() {
+ return retryTimeout;
+ }
+
+ public int getMaxRetries() {
+ return maxRetries;
+ }
+
+ public int getSessionTimeout() {
+ return sessionTimeout;
+ }
+
+ public int getMaxMissedHeartbeats() {
+ return maxMissedHeartbeats;
+ }
+
+ // 智能重试策略 getter 方法
+ public int getInitialRetryDelay() {
+ return initialRetryDelay;
+ }
+
+ public int getMaxRetryDelay() {
+ return maxRetryDelay;
+ }
+
+ public double getBackoffMultiplier() {
+ return backoffMultiplier;
+ }
+
+ public int getMaxInFlightMessages() {
+ return maxInFlightMessages;
+ }
+
+ public int getTotalRetryTimeout() {
+ return totalRetryTimeout;
+ }
+
+ // 持久化存储配置 getter 方法
+ public String getStoreType() {
+ return storeType;
+ }
+
+ public String getStorePath() {
+ return storePath;
+ }
+
+ public long getStoreMaxFileSize() {
+ return storeMaxFileSize;
+ }
+
+ public long getStoreRetentionTime() {
+ return storeRetentionTime;
+ }
+
+ public int getStoreSyncInterval() {
+ return storeSyncInterval;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private boolean enabled = false;
+ private int heartbeatInterval = 5000;
+ private int retryTimeout = 10000;
+ private int maxRetries = 3;
+ private int sessionTimeout = 30000;
+ private int maxMissedHeartbeats = 3;
+
+ // 智能重试策略默认值
+ private int initialRetryDelay = 100; // 初始重试延迟 100ms
+ private int maxRetryDelay = 5000; // 最大重试延迟 5s
+ private double backoffMultiplier = 2.0; // 退避倍数 2.0
+ private int maxInFlightMessages = 100; // 最大 InFlight 消息数 100
+ private int totalRetryTimeout = 30000; // 总重试超时 30s
+
+ // 持久化存储配置默认值
+ private String storeType = "memory"; // 默认内存存储
+ private String storePath = "/tmp/dubbo/reliable"; // 默认存储路径
+ private long storeMaxFileSize = 100 * 1024 * 1024; // 默认 100MB
+ private long storeRetentionTime = 24 * 60 * 60 * 1000; // 默认 24小时
+ private int storeSyncInterval = 1000; // 默认 1s同步间隔
+
+ public Builder enabled(boolean enabled) {
+ this.enabled = enabled;
+ return this;
+ }
+
+ public Builder heartbeatInterval(int interval) {
+ this.heartbeatInterval = interval;
+ return this;
+ }
+
+ public Builder retryTimeout(int timeout) {
+ this.retryTimeout = timeout;
+ return this;
+ }
+
+ public Builder maxRetries(int maxRetries) {
+ this.maxRetries = maxRetries;
+ return this;
+ }
+
+ public Builder sessionTimeout(int sessionTimeout) {
+ this.sessionTimeout = sessionTimeout;
+ return this;
+ }
+
+ public Builder maxMissedHeartbeats(int maxMissedHeartbeats) {
+ this.maxMissedHeartbeats = maxMissedHeartbeats;
+ return this;
+ }
+
+ // 智能重试策略 Builder 方法
+ public Builder initialRetryDelay(int initialRetryDelay) {
+ this.initialRetryDelay = initialRetryDelay;
+ return this;
+ }
+
+ public Builder maxRetryDelay(int maxRetryDelay) {
+ this.maxRetryDelay = maxRetryDelay;
+ return this;
+ }
+
+ public Builder backoffMultiplier(double backoffMultiplier) {
+ this.backoffMultiplier = backoffMultiplier;
+ return this;
+ }
+
+ public Builder maxInFlightMessages(int maxInFlightMessages) {
+ this.maxInFlightMessages = maxInFlightMessages;
+ return this;
+ }
+
+ public Builder totalRetryTimeout(int totalRetryTimeout) {
+ this.totalRetryTimeout = totalRetryTimeout;
+ return this;
+ }
+
+ // 持久化存储配置 Builder 方法
+ public Builder storeType(String storeType) {
+ this.storeType = storeType;
+ return this;
+ }
+
+ public Builder storePath(String storePath) {
+ this.storePath = storePath;
+ return this;
+ }
+
+ public Builder storeMaxFileSize(long storeMaxFileSize) {
+ this.storeMaxFileSize = storeMaxFileSize;
+ return this;
+ }
+
+ public Builder storeRetentionTime(long storeRetentionTime) {
+ this.storeRetentionTime = storeRetentionTime;
+ return this;
+ }
+
+ public Builder storeSyncInterval(int storeSyncInterval) {
+ this.storeSyncInterval = storeSyncInterval;
+ return this;
+ }
+
+ public ReliabilityConfig build() {
+ return new ReliabilityConfig(this);
+ }
+ }
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReliabilityContext.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReliabilityContext.java
new file mode 100644
index 0000000000..c465124466
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/stream/ReliabilityContext.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.rpc.protocol.tri.stream;
+
+import java.util.Map;
+import java.util.function.Consumer;
+
+/**
+ * ReliabilityContext provides business observability APIs for reliable streaming.
+ * This interface allows business logic to monitor and control reliability aspects
+ * of the streaming communication.
+ */
+public interface ReliabilityContext {
+
+ /**
+ * Get the current reliability state of the stream.
+ *
+ * @return current state string (e.g., "HEALTHY", "RECONNECTING", "FAILED")
+ */
+ String getState();
+
+ /**
+ * Get the last acknowledged sequence number.
+ *
+ * @return last acknowledged sequence number, or -1 if reliability is not enabled
+ */
+ long getLastAckedSeq();
+
+ /**
+ * Get the current pending message count.
+ *
+ * @return number of messages waiting for acknowledgement
+ */
+ int getPendingCount();
+
+ /**
+ * Get the current InFlight message count.
+ *
+ * @return number of messages currently in flight
+ */
+ int getInFlightCount();
+
+ /**
+ * Get the session ID for this reliable stream.
+ *
+ * @return session ID, or null if reliability is not enabled
+ */
+ String getSessionId();
+
+ /**
+ * Get the total number of messages sent.
+ *
+ * @return total sent message count
+ */
+ long getTotalSentCount();
+
+ /**
+ * Get the total number of retries attempted.
+ *
+ * @return total retry count
+ */
+ long getTotalRetryCount();
+
+ /**
+ * Get the connection status.
+ *
+ * @return true if connection is active, false otherwise
+ */
+ boolean isConnectionActive();
+
+ /**
+ * Get reliability statistics as a map.
+ *
+ * @return map containing all reliability metrics
+ */
+ Map<String, Object> getStatistics();
+
+ /**
+ * Register a callback for reliability state changes.
+ *
+ * @param callback consumer that receives state change events
+ */
+ void onStateChange(Consumer<String> callback);
+
+ /**
+ * Register a callback for connection recovery events.
+ *
+ * @param callback consumer that receives recovery events
+ */
+ void onRecovery(Consumer<String> callback);
+
+ /**
+ * Register a callback for retry events.
+ *
+ * @param callback consumer that receives retry events with sequence number
+ */
+ void onRetry(Consumer<Long> callback);
+
+ /**
+ * Manually trigger a retry for a specific message.
+ *
+ * @param sequence the sequence number to retry
+ * @return true if retry was triggered, false if message not found or not retryable
+ */
+ boolean retryMessage(long sequence);
+
+ /**
+ * Manually trigger connection recovery.
+ *
+ * @return true if recovery was triggered, false if recovery is already in progress
+ */
+ boolean triggerRecovery();
+
+ /**
+ * Set the maximum InFlight message limit dynamically.
+ *
+ * @param limit the new InFlight limit
+ */
+ void setInFlightLimit(int limit);
+
+ /**
+ * Get the current retry configuration.
+ *
+ * @return ReliabilityConfig instance
+ */
+ ReliabilityConfig getRetryConfig();
+}
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.protocol.tri.server.ServerSequenceStore b/dubbo-rpc/dubbo-rpc-triple/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.protocol.tri.server.ServerSequenceStore
new file mode 100644
index 0000000000..a616f7d85b
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.protocol.tri.server.ServerSequenceStore
@@ -0,0 +1,2 @@
+memory=org.apache.dubbo.rpc.protocol.tri.server.InMemoryServerSequenceStore
+file=org.apache.dubbo.rpc.protocol.tri.server.FileServerSequenceStore
\ No newline at end of file
diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.protocol.tri.store.PendingMessageStore b/dubbo-rpc/dubbo-rpc-triple/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.protocol.tri.store.PendingMessageStore
new file mode 100644
index 0000000000..e84fd6a901
--- /dev/null
+++ b/dubbo-rpc/dubbo-rpc-triple/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.protocol.tri.store.PendingMessageStore
@@ -0,0 +1,2 @@
+memory=org.apache.dubbo.rpc.protocol.tri.store.InMemoryPendingMessageStore
+file=org.apache.dubbo.rpc.protocol.tri.store.FilePendingMessageStore
\ No newline at end of file