You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by li...@apache.org on 2024/09/13 03:45:27 UTC
(rocketmq) branch develop updated: [ISSUE #8599] Fix send fail with receiving GO_AWAY when rolling update proxy and add channel id in logs (#8685)
This is an automated email from the ASF dual-hosted git repository.
lizhimin pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
The following commit(s) were added to refs/heads/develop by this push:
new 7748040f41 [ISSUE #8599] Fix send fail with receiving GO_AWAY when rolling update proxy and add channel id in logs (#8685)
7748040f41 is described below
commit 7748040f41a69a7e18e2acb55c03005f539a4ba2
Author: qianye <37...@users.noreply.github.com>
AuthorDate: Fri Sep 13 11:45:21 2024 +0800
[ISSUE #8599] Fix send fail with receiving GO_AWAY when rolling update proxy and add channel id in logs (#8685)
---
.../remoting/netty/NettyRemotingAbstract.java | 8 +--
.../remoting/netty/NettyRemotingClient.java | 61 ++++++++++++----------
2 files changed, 37 insertions(+), 32 deletions(-)
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index 9f3136195b..ffa3726059 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -39,8 +39,8 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import javax.annotation.Nullable;
import org.apache.rocketmq.common.AbortProcessException;
@@ -393,7 +393,7 @@ public abstract class NettyRemotingAbstract {
responseFuture.release();
}
} else {
- log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+ log.warn("receive response, cmd={}, but not matched any request, address={}, channelId={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), ctx.channel().id());
}
}
@@ -560,13 +560,13 @@ public abstract class NettyRemotingAbstract {
return;
}
requestFail(opaque);
- log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel));
+ log.warn("send a request command to channel <{}>, channelId={}, failed.", RemotingHelper.parseChannelRemoteAddr(channel), channel.id());
});
return future;
} catch (Exception e) {
responseTable.remove(opaque);
responseFuture.release();
- log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e);
+ log.warn("send a request command to channel <{}> channelId={} Exception", RemotingHelper.parseChannelRemoteAddr(channel), channel.id(), e);
future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e));
return future;
}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
index 41976122b2..ef9762ddc6 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
@@ -49,7 +49,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.security.cert.CertificateException;
-import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -416,14 +415,14 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
boolean removeItemFromTable = true;
final ChannelWrapper prevCW = this.channelTables.get(addrRemote);
- LOGGER.info("closeChannel: begin close the channel[{}] Found: {}", addrRemote, prevCW != null);
+ LOGGER.info("closeChannel: begin close the channel[addr={}, id={}] Found: {}", addrRemote, channel.id(), prevCW != null);
if (null == prevCW) {
- LOGGER.info("closeChannel: the channel[{}] has been removed from the channel table before", addrRemote);
+ LOGGER.info("closeChannel: the channel[addr={}, id={}] has been removed from the channel table before", addrRemote, channel.id());
removeItemFromTable = false;
} else if (prevCW.isWrapperOf(channel)) {
- LOGGER.info("closeChannel: the channel[{}] has been closed before, and has been created again, nothing to do.",
- addrRemote);
+ LOGGER.info("closeChannel: the channel[addr={}, id={}] has been closed before, and has been created again, nothing to do.",
+ addrRemote, channel.id());
removeItemFromTable = false;
}
@@ -432,7 +431,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
if (channelWrapper != null && channelWrapper.tryClose(channel)) {
this.channelTables.remove(addrRemote);
}
- LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote);
+ LOGGER.info("closeChannel: the channel[addr={}, id={}] was removed from channel table", addrRemote, channel.id());
}
RemotingHelper.closeChannel(channel);
@@ -471,7 +470,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
}
if (null == prevCW) {
- LOGGER.info("eventCloseChannel: the channel[{}] has been removed from the channel table before", addrRemote);
+ LOGGER.info("eventCloseChannel: the channel[addr={}, id={}] has been removed from the channel table before", RemotingHelper.parseChannelRemoteAddr(channel), channel.id());
removeItemFromTable = false;
}
@@ -480,11 +479,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
if (channelWrapper != null && channelWrapper.tryClose(channel)) {
this.channelTables.remove(addrRemote);
}
- LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote);
+ LOGGER.info("closeChannel: the channel[addr={}, id={}] was removed from channel table", addrRemote, channel.id());
RemotingHelper.closeChannel(channel);
}
} catch (Exception e) {
- LOGGER.error("closeChannel: close the channel exception", e);
+ LOGGER.error("closeChannel: close the channel[id={}] exception", channel.id(), e);
} finally {
this.lockChannelTables.unlock();
}
@@ -562,9 +561,9 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
boolean shouldClose = left > MIN_CLOSE_TIMEOUT_MILLIS || left > timeoutMillis / 4;
if (nettyClientConfig.isClientCloseSocketIfTimeout() && shouldClose) {
this.closeChannel(addr, channel);
- LOGGER.warn("invokeSync: close socket because of timeout, {}ms, {}", timeoutMillis, channelRemoteAddr);
+ LOGGER.warn("invokeSync: close socket because of timeout, {}ms, channel[addr={}, id={}]", timeoutMillis, channelRemoteAddr, channel.id());
}
- LOGGER.warn("invokeSync: wait response timeout exception, the channel[{}]", channelRemoteAddr);
+ LOGGER.warn("invokeSync: wait response timeout exception, the channel[addr={}, id={}]", channelRemoteAddr, channel.id());
throw e;
}
} else {
@@ -819,10 +818,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
RemotingCommand response = responseFuture.getResponseCommand();
if (response.getCode() == ResponseCode.GO_AWAY) {
if (nettyClientConfig.isEnableReconnectForGoAway()) {
+ LOGGER.info("Receive go away from channelId={}, channel={}", channel.id(), channel);
ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> {
try {
- if (channelWrapper0.reconnect()) {
- LOGGER.info("Receive go away from channel {}, recreate the channel", channel0);
+ if (channelWrapper0.reconnect(channel0)) {
+ LOGGER.info("Receive go away from channelId={}, channel={}, recreate the channelId={}", channel0.id(), channel0, channelWrapper0.getChannel().id());
channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0);
}
} catch (Throwable t) {
@@ -830,10 +830,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
}
return channelWrapper0;
});
- if (channelWrapper != null) {
+ if (channelWrapper != null && !channelWrapper.isWrapperOf(channel)) {
if (nettyClientConfig.isEnableTransparentRetry()) {
RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader());
retryRequest.setBody(request.getBody());
+ retryRequest.setExtFields(request.getExtFields());
if (channelWrapper.isOK()) {
long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS);
stopwatch.stop();
@@ -865,6 +866,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
return future;
}
}
+ } else {
+ LOGGER.warn("invokeImpl receive GO_AWAY, channelWrapper is null or channel is the same in wrapper, channelId={}", channel.id());
}
}
}
@@ -1002,7 +1005,6 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
// only affected by sync or async request, oneway is not included.
private ChannelFuture channelToClose;
private long lastResponseTime;
- private volatile long lastReconnectTimestamp = 0L;
private final String channelAddress;
public ChannelWrapper(String address, ChannelFuture channelFuture) {
@@ -1021,10 +1023,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
}
public boolean isWrapperOf(Channel channel) {
- if (this.channelFuture.channel() != null && this.channelFuture.channel() == channel) {
- return true;
- }
- return false;
+ return this.channelFuture.channel() != null && this.channelFuture.channel() == channel;
}
private Channel getChannel() {
@@ -1052,20 +1051,27 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
return channelAddress;
}
- public boolean reconnect() {
+ public boolean reconnect(Channel channel) {
+ if (!isWrapperOf(channel)) {
+ LOGGER.warn("channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}",getChannel().id(), channel.id());
+ return false;
+ }
if (lock.writeLock().tryLock()) {
try {
- if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) {
+ if (isWrapperOf(channel)) {
channelToClose = channelFuture;
String[] hostAndPort = getHostAndPort(channelAddress);
channelFuture = fetchBootstrap(channelAddress)
.connect(hostAndPort[0], Integer.parseInt(hostAndPort[1]));
- lastReconnectTimestamp = System.currentTimeMillis();
return true;
+ } else {
+ LOGGER.warn("channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}",getChannel().id(), channel.id());
}
} finally {
lock.writeLock().unlock();
}
+ } else {
+ LOGGER.warn("channelWrapper reconnect try lock fail, now channelId={}", getChannel().id());
}
return false;
}
@@ -1152,7 +1158,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
- LOGGER.info("NETTY CLIENT PIPELINE: ACTIVE, {}", remoteAddress);
+ LOGGER.info("NETTY CLIENT PIPELINE: ACTIVE, {}, channelId={}", remoteAddress, ctx.channel().id());
super.channelActive(ctx);
if (NettyRemotingClient.this.channelEventListener != null) {
@@ -1175,7 +1181,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
@Override
public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception {
final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
- LOGGER.info("NETTY CLIENT PIPELINE: CLOSE {}", remoteAddress);
+ LOGGER.info("NETTY CLIENT PIPELINE: CLOSE channel[addr={}, id={}]", remoteAddress, ctx.channel().id());
closeChannel(ctx.channel());
super.close(ctx, promise);
NettyRemotingClient.this.failFast(ctx.channel());
@@ -1187,7 +1193,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
- LOGGER.info("NETTY CLIENT PIPELINE: channelInactive, the channel[{}]", remoteAddress);
+ LOGGER.info("NETTY CLIENT PIPELINE: channelInactive, the channel[addr={}, id={}]", remoteAddress, ctx.channel().id());
closeChannel(ctx.channel());
super.channelInactive(ctx);
}
@@ -1198,7 +1204,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
IdleStateEvent event = (IdleStateEvent) evt;
if (event.state().equals(IdleState.ALL_IDLE)) {
final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
- LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception [{}]", remoteAddress);
+ LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception channel[addr={}, id={}]", remoteAddress, ctx.channel().id());
closeChannel(ctx.channel());
if (NettyRemotingClient.this.channelEventListener != null) {
NettyRemotingClient.this
@@ -1213,8 +1219,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
- LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught {}", remoteAddress);
- LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught exception.", cause);
+ LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught channel[addr={}, id={}]", remoteAddress, ctx.channel().id(), cause);
closeChannel(ctx.channel());
if (NettyRemotingClient.this.channelEventListener != null) {
NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress, ctx.channel()));