You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by zh...@apache.org on 2022/12/09 11:53:41 UTC
[rocketmq] branch develop updated (774bc6752 -> bdf5c95ce)
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a change to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
from 774bc6752 optimize LagCalculationIT
new 934f5b657 [ISSUE #5406] Add processor for remoting messaging module
new 4b9816e4a [ISSUE #5406] Support getConsumerIdList
new b887313bf [ISSUE #5406] Add unit test
new 56551b596 [ISSUE #5485] client connection management
new 6c8233043 [ISSUE #5485] test cases for client connection management
new 48673af76 [ISSUE #5486] Add remoting server
new 8a23c54a9 [ISSUE #5486] polish clean expire request in remoting server
new 4c0e06749 [ISSUE #5486] polish MultiProtocolRemotingServer
new 7e916e3db [ISSUE #5406] support transaction message for remoting proxy
new 832fc57a0 [ISSUE #5486] polish tls config; add tcnative dependency
new 21a01e3c6 [ISSUE #5406] use remotingAccessAddr and ListenPort in topicRoute for remoting proxy
new 6351e1957 [ISSUE #5485] polish channel management which is been synced from other proxy
new 96bab46bd [ISSUE #5485] polish channel management
new 834287af0 [ISSUE #5485] add test cases for channel management
new 29c4122d9 [ISSUE #5486] Add AuthenticationPipeline
new 561fc2e73 [ISSUE #5392] Remove unused method
new 274a53132 [ISSUE #5406] Fix brokerName for SEND_MESSAGE_V2
new 9897e74a6 [ISSUE #5406] Overwrite sysFlag to broker
new 416427de1 [ISSUE #5392] Adapt for logging and module refector
new 1e71cd118 [ISSUE #5406] Fix PullMessageActivityTest
new 23c8efd80 [ISSUE #5485] Fix GrpcBaseIT
new b713aebfa [ISSUE #5485] Fix by code review
new 8c9a9d36b [ISSUE #5485] Remove ConsumerManagerInterface and ProducerManagerInterface
new 4b311ff4d [ISSUE #5485] Use local address, remoting port and grpc port to build unique local proxy Id
new 373c1bd13 [ISSUE #5485] Fix merge code issue
new bdf5c95ce [ISSUE #5485] Fix ClusterGrpcIT
The 26 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../rocketmq/broker/client/ConsumerManager.java | 2 +-
.../common/attribute/TopicMessageType.java | 17 +
.../rocketmq/common/sysflag/PullSysFlag.java | 4 +
...r.java => FutureTaskExtThreadPoolExecutor.java} | 31 +-
.../rocketmq/common/thread/ThreadPoolMonitor.java | 2 +-
pom.xml | 6 +
proxy/pom.xml | 4 +
.../org/apache/rocketmq/proxy/ProxyStartup.java | 4 +
.../proxy/common/channel/ChannelHelper.java | 49 +++
.../rocketmq/proxy/common/utils/FutureUtils.java | 6 +
.../apache/rocketmq/proxy/config/ProxyConfig.java | 321 +++++++++++++++++-
.../rocketmq/proxy/grpc/GrpcServerBuilder.java | 6 +-
.../proxy/grpc/v2/DefaultGrpcMessingActivity.java | 3 +-
.../proxy/grpc/v2/channel/GrpcChannelManager.java | 7 +-
.../proxy/grpc/v2/channel/GrpcClientChannel.java | 71 +++-
.../proxy/grpc/v2/client/ClientActivity.java | 54 ++-
.../grpc/v2/common/GrpcClientSettingsManager.java | 57 +++-
.../proxy/processor/AbstractProcessor.java | 19 --
.../rocketmq/proxy/processor/ClientProcessor.java | 5 +
.../proxy/processor/DefaultMessagingProcessor.java | 19 ++
.../proxy/processor/MessagingProcessor.java | 8 +
.../proxy/processor/ProducerProcessor.java | 2 +-
.../proxy/processor/ReceiptHandleProcessor.java | 6 +
.../RequestBrokerProcessor.java} | 26 +-
.../channel/ChannelExtendAttributeGetter.java} | 6 +-
.../processor/channel/ChannelProtocolType.java | 24 +-
.../proxy/processor/channel/RemoteChannel.java | 116 +++++++
.../channel/RemoteChannelConverter.java} | 6 +-
.../processor/channel/RemoteChannelSerializer.java | 65 ++++
.../proxy/remoting/ClientHousekeepingService.java | 22 +-
.../remoting/MultiProtocolRemotingServer.java | 82 +++++
.../proxy/remoting/MultiProtocolTlsHelper.java | 113 +++++++
.../proxy/remoting/RemotingProtocolServer.java | 368 +++++++++++++++++++++
.../RemotingProxyOutClient.java} | 9 +-
.../activity/AbstractRemotingActivity.java | 178 ++++++++++
.../activity/AckMessageActivity.java} | 22 +-
.../activity/ChangeInvisibleTimeActivity.java} | 22 +-
.../remoting/activity/ClientManagerActivity.java | 179 ++++++++++
.../remoting/activity/ConsumerManagerActivity.java | 125 +++++++
.../remoting/activity/GetTopicRouteActivity.java | 72 ++++
.../remoting/activity/PopMessageActivity.java | 41 +++
.../remoting/activity/PullMessageActivity.java | 63 ++++
.../remoting/activity/SendMessageActivity.java | 90 +++++
.../remoting/activity/TransactionActivity.java | 68 ++++
.../proxy/remoting/channel/RemotingChannel.java | 231 +++++++++++++
.../remoting/channel/RemotingChannelManager.java | 141 ++++++++
.../common/RemotingConverter.java} | 41 +--
.../remoting/pipeline/AuthenticationPipeline.java | 46 +++
.../proxy/remoting/pipeline/RequestPipeline.java | 20 +-
.../proxy/remoting/protocol/ProtocolHandler.java | 14 +-
.../protocol/ProtocolNegotiationHandler.java | 61 ++++
.../http2proxy/Http2ProtocolProxyHandler.java | 119 +++++++
.../http2proxy/Http2ProxyBackendHandler.java | 67 ++++
.../http2proxy/Http2ProxyFrontendHandler.java | 78 +++++
.../protocol/remoting/RemotingProtocolHandler.java | 60 ++++
.../proxy/service/ClusterServiceManager.java | 29 +-
.../proxy/service/LocalServiceManager.java | 11 +-
.../rocketmq/proxy/service/ServiceManager.java | 3 +
.../rocketmq/proxy/service/admin/AdminService.java | 22 +-
.../proxy/service/admin/DefaultAdminService.java | 146 ++++++++
.../proxy/service/channel/SimpleChannel.java | 13 +
.../service/client/ClusterConsumerManager.java | 69 ++++
.../service/message/ClusterMessageService.java | 34 +-
.../proxy/service/message/LocalMessageService.java | 12 +
.../proxy/service/message/MessageService.java | 6 +
.../sysmessage/AbstractSystemMessageSyncer.java | 182 ++++++++++
.../proxy/service/sysmessage/HeartbeatSyncer.java | 230 +++++++++++++
.../service/sysmessage/HeartbeatSyncerData.java | 176 ++++++++++
.../proxy/service/sysmessage/HeartbeatType.java | 6 +-
.../rocketmq/proxy/grpc/v2/BaseActivityTest.java | 2 +-
.../grpc/v2/channel/GrpcClientChannelTest.java | 82 +++++
.../proxy/processor/channel/RemoteChannelTest.java | 50 +++
.../activity/AbstractRemotingActivityTest.java | 200 +++++++++++
.../remoting/activity/PullMessageActivityTest.java | 165 +++++++++
.../remoting/activity/SendMessageActivityTest.java | 102 ++++++
.../channel/RemotingChannelManagerTest.java | 162 +++++++++
.../remoting/channel/RemotingChannelTest.java | 80 +++++
.../service/admin/DefaultAdminServiceTest.java | 103 ++++++
.../mqclient/ProxyClientRemotingProcessorTest.java | 5 +-
.../service/sysmessage/HeartbeatSyncerTest.java | 319 ++++++++++++++++++
.../remoting/netty/NettyRemotingServer.java | 68 +++-
.../protocol/header/SendMessageRequestHeader.java | 74 -----
.../apache/rocketmq/test/grpc/v2/GrpcBaseIT.java | 1 +
83 files changed, 5323 insertions(+), 307 deletions(-)
copy common/src/main/java/org/apache/rocketmq/common/thread/{ThreadPoolQueueSizeMonitor.java => FutureTaskExtThreadPoolExecutor.java} (52%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/common/channel/ChannelHelper.java
copy proxy/src/main/java/org/apache/rocketmq/proxy/{service/channel/InvocationContext.java => processor/RequestBrokerProcessor.java} (52%)
copy proxy/src/main/java/org/apache/rocketmq/proxy/{config/ConfigFile.java => processor/channel/ChannelExtendAttributeGetter.java} (85%)
copy common/src/main/java/org/apache/rocketmq/common/message/MessageRequestMode.java => proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelProtocolType.java (73%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java
copy proxy/src/main/java/org/apache/rocketmq/proxy/{config/ConfigFile.java => processor/channel/RemoteChannelConverter.java} (85%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java
copy namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java => proxy/src/main/java/org/apache/rocketmq/proxy/remoting/ClientHousekeepingService.java (67%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
copy proxy/src/main/java/org/apache/rocketmq/proxy/{service/channel/InvocationContextInterface.java => remoting/RemotingProxyOutClient.java} (75%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
copy proxy/src/main/java/org/apache/rocketmq/proxy/{service/mqclient/DoNothingClientRemotingProcessor.java => remoting/activity/AckMessageActivity.java} (56%)
copy proxy/src/main/java/org/apache/rocketmq/proxy/{service/mqclient/DoNothingClientRemotingProcessor.java => remoting/activity/ChangeInvisibleTimeActivity.java} (55%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
copy proxy/src/main/java/org/apache/rocketmq/proxy/{grpc/GrpcServer.java => remoting/common/RemotingConverter.java} (53%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java
copy remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRequestProcessor.java => proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/RequestPipeline.java (66%)
copy test/src/main/java/org/apache/rocketmq/test/clientinterface/MQConsumer.java => proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolHandler.java (70%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolNegotiationHandler.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java
copy remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ProducerTableInfo.java => proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java (59%)
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
copy acl/src/main/java/org/apache/rocketmq/acl/AccessResource.java => proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatType.java (87%)
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelTest.java
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManagerTest.java
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java
create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
[rocketmq] 04/26: [ISSUE #5485] client connection management
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 56551b596b3f2f7b3e16e5f271200bceaf28734a
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Tue Nov 8 17:18:24 2022 +0800
[ISSUE #5485] client connection management
---
.../rocketmq/broker/client/ConsumerManager.java | 13 +-
.../broker/client/ConsumerManagerInterface.java | 60 ++++++
.../rocketmq/broker/client/ProducerManager.java | 10 +
.../broker/client/ProducerManagerInterface.java | 44 ++++
.../proxy/common/channel/ChannelHelper.java | 49 +++++
.../apache/rocketmq/proxy/config/ProxyConfig.java | 49 +++++
.../proxy/grpc/v2/DefaultGrpcMessingActivity.java | 2 +-
.../proxy/grpc/v2/channel/GrpcChannelManager.java | 7 +-
.../proxy/grpc/v2/channel/GrpcClientChannel.java | 71 ++++++-
.../proxy/grpc/v2/client/ClientActivity.java | 76 ++++++-
.../grpc/v2/common/GrpcClientSettingsManager.java | 9 +
.../rocketmq/proxy/processor/ClientProcessor.java | 5 +
.../proxy/processor/DefaultMessagingProcessor.java | 5 +
.../proxy/processor/MessagingProcessor.java | 2 +
.../proxy/processor/ReceiptHandleProcessor.java | 5 +
.../channel/ChannelExtendAttributeGetter.java | 23 +++
.../processor/channel/ChannelProtocolType.java | 35 ++++
.../proxy/processor/channel/RemoteChannel.java | 98 +++++++++
.../processor/channel/RemoteChannelConverter.java | 23 +++
.../processor/channel/RemoteChannelSerializer.java | 65 ++++++
.../proxy/remoting/ClientHousekeepingService.java | 53 +++++
.../proxy/remoting/RemotingProtocolServer.java | 72 +++++++
.../proxy/remoting/RemotingProxyOutClient.java | 27 +++
.../remoting/activity/ClientManagerActivity.java | 178 ++++++++++++++++
.../proxy/remoting/channel/RemotingChannel.java | 224 +++++++++++++++++++++
.../remoting/channel/RemotingChannelManager.java | 142 +++++++++++++
.../proxy/remoting/common/RemotingConverter.java | 74 +++++++
.../proxy/service/ClusterServiceManager.java | 17 +-
.../proxy/service/LocalServiceManager.java | 11 +-
.../rocketmq/proxy/service/ServiceManager.java | 11 +-
.../rocketmq/proxy/service/admin/AdminService.java | 32 +++
.../proxy/service/admin/DefaultAdminService.java | 142 +++++++++++++
.../proxy/service/channel/SimpleChannel.java | 8 +
.../service/client/ClusterConsumerManager.java | 70 +++++++
.../sysmessage/AbstractSystemMessageSyncer.java | 190 +++++++++++++++++
.../proxy/service/sysmessage/HeartbeatSyncer.java | 224 +++++++++++++++++++++
.../service/sysmessage/HeartbeatSyncerData.java | 164 +++++++++++++++
.../proxy/service/sysmessage/HeartbeatType.java | 23 +++
.../mqclient/ProxyClientRemotingProcessorTest.java | 5 +-
39 files changed, 2285 insertions(+), 33 deletions(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
index 5f95ac1af..0582ce75e 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
@@ -64,6 +64,7 @@ public class ConsumerManager {
this.subscriptionExpiredTimeout = brokerConfig.getSubscriptionExpiredTimeout();
}
+ @Override
public ClientChannelInfo findChannel(final String group, final String clientId) {
ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
if (consumerGroupInfo != null) {
@@ -72,6 +73,7 @@ public class ConsumerManager {
return null;
}
+ @Override
public ClientChannelInfo findChannel(final String group, final Channel channel) {
ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
if (consumerGroupInfo != null) {
@@ -80,6 +82,7 @@ public class ConsumerManager {
return null;
}
+ @Override
public SubscriptionData findSubscriptionData(final String group, final String topic) {
return findSubscriptionData(group, topic, true);
}
@@ -107,6 +110,7 @@ public class ConsumerManager {
return this.consumerTable;
}
+ @Override
public ConsumerGroupInfo getConsumerGroupInfo(final String group) {
return getConsumerGroupInfo(group, false);
}
@@ -119,6 +123,7 @@ public class ConsumerManager {
return consumerGroupInfo;
}
+ @Override
public int findSubscriptionDataCount(final String group) {
ConsumerGroupInfo consumerGroupInfo = this.getConsumerGroupInfo(group);
if (consumerGroupInfo != null) {
@@ -128,6 +133,7 @@ public class ConsumerManager {
return 0;
}
+ @Override
public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
boolean removed = false;
Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
@@ -172,6 +178,7 @@ public class ConsumerManager {
isNotifyConsumerIdsChangedEnable, true);
}
+ @Override
public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription) {
@@ -202,11 +209,12 @@ public class ConsumerManager {
this.brokerStatsManager.incConsumerRegisterTime((int) (System.currentTimeMillis() - start));
}
- callConsumerIdsChangeListener(ConsumerGroupEvent.REGISTER, group, subList);
+ callConsumerIdsChangeListener(ConsumerGroupEvent.REGISTER, group, subList, clientChannelInfo);
return r1 || r2;
}
+ @Override
public void unregisterConsumer(final String group, final ClientChannelInfo clientChannelInfo,
boolean isNotifyConsumerIdsChangedEnable) {
ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
@@ -252,6 +260,7 @@ public class ConsumerManager {
}
}
+ @Override
public void scanNotActiveChannel() {
Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
while (it.hasNext()) {
@@ -286,6 +295,7 @@ public class ConsumerManager {
removeExpireConsumerGroupInfo();
}
+ @Override
public HashSet<String> queryTopicConsumeByWho(final String topic) {
HashSet<String> groups = new HashSet<>();
Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
@@ -300,6 +310,7 @@ public class ConsumerManager {
return groups;
}
+ @Override
public void appendConsumerIdsChangeListener(ConsumerIdsChangeListener listener) {
consumerIdsChangeListenerList.add(listener);
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java
new file mode 100644
index 000000000..895a2e491
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.client;
+
+import io.netty.channel.Channel;
+import java.util.Set;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+
+public interface ConsumerManagerInterface {
+
+ ClientChannelInfo findChannel(String group, String clientId);
+
+ ClientChannelInfo findChannel(String group, Channel channel);
+
+ SubscriptionData findSubscriptionData(String group, String topic);
+
+ ConsumerGroupInfo getConsumerGroupInfo(String group);
+
+ int findSubscriptionDataCount(String group);
+
+ boolean doChannelCloseEvent(String remoteAddr, Channel channel);
+
+ default boolean registerConsumer(String group, ClientChannelInfo clientChannelInfo,
+ ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
+ Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {
+ return registerConsumer(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList,
+ isNotifyConsumerIdsChangedEnable, true);
+ }
+
+ boolean registerConsumer(String group, ClientChannelInfo clientChannelInfo,
+ ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
+ Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription);
+
+ void unregisterConsumer(String group, ClientChannelInfo clientChannelInfo,
+ boolean isNotifyConsumerIdsChangedEnable);
+
+ void scanNotActiveChannel();
+
+ Set<String> queryTopicConsumeByWho(String topic);
+
+ void appendConsumerIdsChangeListener(ConsumerIdsChangeListener listener);
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
index 52d67bf28..047aa8be9 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
@@ -54,10 +54,12 @@ public class ProducerManager {
this.brokerStatsManager = brokerStatsManager;
}
+ @Override
public int groupSize() {
return this.groupChannelTable.size();
}
+ @Override
public boolean groupOnline(String group) {
Map<Channel, ClientChannelInfo> channels = this.groupChannelTable.get(group);
return channels != null && !channels.isEmpty();
@@ -67,6 +69,7 @@ public class ProducerManager {
return groupChannelTable;
}
+ @Override
public ProducerTableInfo getProducerTable() {
Map<String, List<ProducerInfo>> map = new HashMap<>();
for (String group : this.groupChannelTable.keySet()) {
@@ -94,6 +97,7 @@ public class ProducerManager {
return new ProducerTableInfo(map);
}
+ @Override
public void scanNotActiveChannel() {
Iterator<Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>>> iterator = this.groupChannelTable.entrySet().iterator();
@@ -129,6 +133,7 @@ public class ProducerManager {
}
}
+ @Override
public synchronized boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
boolean removed = false;
if (channel != null) {
@@ -160,6 +165,7 @@ public class ProducerManager {
return removed;
}
+ @Override
public synchronized void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) {
ClientChannelInfo clientChannelInfoFound = null;
@@ -183,6 +189,7 @@ public class ProducerManager {
}
}
+ @Override
public synchronized void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) {
ConcurrentHashMap<Channel, ClientChannelInfo> channelTable = this.groupChannelTable.get(group);
if (null != channelTable && !channelTable.isEmpty()) {
@@ -202,6 +209,7 @@ public class ProducerManager {
}
}
+ @Override
public Channel getAvailableChannel(String groupId) {
if (groupId == null) {
return null;
@@ -242,6 +250,7 @@ public class ProducerManager {
return lastActiveChannel;
}
+ @Override
public Channel findChannel(String clientId) {
return clientChannelTable.get(clientId);
}
@@ -257,6 +266,7 @@ public class ProducerManager {
}
}
+ @Override
public void appendProducerChangeListener(ProducerChangeListener producerChangeListener) {
producerChangeListenerList.add(producerChangeListener);
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java
new file mode 100644
index 000000000..3f2ece7cd
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.client;
+
+import io.netty.channel.Channel;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
+
+public interface ProducerManagerInterface {
+
+ int groupSize();
+
+ boolean groupOnline(String group);
+
+ ProducerTableInfo getProducerTable();
+
+ void scanNotActiveChannel();
+
+ boolean doChannelCloseEvent(String remoteAddr, Channel channel);
+
+ void registerProducer(String group, ClientChannelInfo clientChannelInfo);
+
+ void unregisterProducer(String group, ClientChannelInfo clientChannelInfo);
+
+ Channel getAvailableChannel(String groupId);
+
+ Channel findChannel(String clientId);
+
+ void appendProducerChangeListener(ProducerChangeListener producerChangeListener);
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/channel/ChannelHelper.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/channel/ChannelHelper.java
new file mode 100644
index 000000000..dd15c85fb
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/channel/ChannelHelper.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common.channel;
+
+import io.netty.channel.Channel;
+import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;
+import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
+import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
+import org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;
+
+public class ChannelHelper {
+
+ /**
+ * judge channel is sync from other proxy or not
+ *
+ * @param channel channel
+ * @return true if is sync from other proxy
+ */
+ public static boolean isRemote(Channel channel) {
+ return channel instanceof RemoteChannel;
+ }
+
+ public static ChannelProtocolType getChannelProtocolType(Channel channel) {
+ if (channel instanceof GrpcClientChannel) {
+ return ChannelProtocolType.GRPC_V2;
+ } else if (channel instanceof RemotingChannel) {
+ return ChannelProtocolType.REMOTING;
+ } else if (channel instanceof RemoteChannel) {
+ RemoteChannel remoteChannel = (RemoteChannel) channel;
+ return remoteChannel.getType();
+ }
+ return ChannelProtocolType.UNKNOWN;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
index cbedc3c50..0efca05b4 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
@@ -34,6 +34,8 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.ProxyMode;
+import org.apache.rocketmq.proxy.common.ProxyException;
+import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
public class ProxyConfig implements ConfigFile {
private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
@@ -55,6 +57,12 @@ public class ProxyConfig implements ConfigFile {
private String proxyClusterName = DEFAULT_CLUSTER_NAME;
private String proxyName = StringUtils.isEmpty(localHostName) ? "DEFAULT_PROXY" : localHostName;
+ private String localServeAddr = "";
+
+ private String systemTopicClusterName = "";
+ private int heartbeatSyncerThreadPoolNums = 4;
+ private int heartbeatSyncerThreadPoolQueueCapacity = 100;
+
/**
* configuration for ThreadPoolMonitor
*/
@@ -204,6 +212,15 @@ public class ProxyConfig implements ConfigFile {
@Override
public void initData() {
parseDelayLevel();
+ if (StringUtils.isEmpty(localServeAddr)) {
+ this.localServeAddr = RemotingUtil.getLocalAddress();
+ }
+ if (StringUtils.isBlank(localServeAddr)) {
+ throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "get local serve ip failed");
+ }
+ if (StringUtils.isBlank(systemTopicClusterName)) {
+ this.systemTopicClusterName = this.rocketMQClusterName;
+ }
}
public int computeDelayLevel(long timeMillis) {
@@ -267,6 +284,38 @@ public class ProxyConfig implements ConfigFile {
this.proxyName = proxyName;
}
+ public String getLocalServeAddr() {
+ return localServeAddr;
+ }
+
+ public void setLocalServeAddr(String localServeAddr) {
+ this.localServeAddr = localServeAddr;
+ }
+
+ public String getSystemTopicClusterName() {
+ return systemTopicClusterName;
+ }
+
+ public void setSystemTopicClusterName(String systemTopicClusterName) {
+ this.systemTopicClusterName = systemTopicClusterName;
+ }
+
+ public int getHeartbeatSyncerThreadPoolNums() {
+ return heartbeatSyncerThreadPoolNums;
+ }
+
+ public void setHeartbeatSyncerThreadPoolNums(int heartbeatSyncerThreadPoolNums) {
+ this.heartbeatSyncerThreadPoolNums = heartbeatSyncerThreadPoolNums;
+ }
+
+ public int getHeartbeatSyncerThreadPoolQueueCapacity() {
+ return heartbeatSyncerThreadPoolQueueCapacity;
+ }
+
+ public void setHeartbeatSyncerThreadPoolQueueCapacity(int heartbeatSyncerThreadPoolQueueCapacity) {
+ this.heartbeatSyncerThreadPoolQueueCapacity = heartbeatSyncerThreadPoolQueueCapacity;
+ }
+
public boolean isEnablePrintJstack() {
return enablePrintJstack;
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
index 81a819007..f30519d74 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
@@ -78,7 +78,7 @@ public class DefaultGrpcMessingActivity extends AbstractStartAndShutdown impleme
protected void init(MessagingProcessor messagingProcessor) {
this.grpcClientSettingsManager = new GrpcClientSettingsManager(messagingProcessor);
- this.grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService());
+ this.grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService(), this.grpcClientSettingsManager);
this.receiptHandleProcessor = new ReceiptHandleProcessor(messagingProcessor);
this.receiveMessageActivity = new ReceiveMessageActivity(messagingProcessor, receiptHandleProcessor, grpcClientSettingsManager, grpcChannelManager);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java
index 97a0ae6da..fb6df2562 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java
@@ -30,12 +30,14 @@ import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
import org.apache.rocketmq.remoting.protocol.ResponseCode;
public class GrpcChannelManager implements StartAndShutdown {
private final ProxyRelayService proxyRelayService;
+ private final GrpcClientSettingsManager grpcClientSettingsManager;
protected final ConcurrentMap<String, GrpcClientChannel> clientIdChannelMap = new ConcurrentHashMap<>();
protected final AtomicLong nonceIdGenerator = new AtomicLong(0);
@@ -45,8 +47,9 @@ public class GrpcChannelManager implements StartAndShutdown {
new ThreadFactoryImpl("GrpcChannelManager_")
);
- public GrpcChannelManager(ProxyRelayService proxyRelayService) {
+ public GrpcChannelManager(ProxyRelayService proxyRelayService, GrpcClientSettingsManager grpcClientSettingsManager) {
this.proxyRelayService = proxyRelayService;
+ this.grpcClientSettingsManager = grpcClientSettingsManager;
}
protected void init() {
@@ -58,7 +61,7 @@ public class GrpcChannelManager implements StartAndShutdown {
public GrpcClientChannel createChannel(ProxyContext ctx, String clientId) {
return this.clientIdChannelMap.computeIfAbsent(clientId,
- k -> new GrpcClientChannel(proxyRelayService, this, ctx, clientId));
+ k -> new GrpcClientChannel(proxyRelayService, grpcClientSettingsManager, this, ctx, clientId));
}
public GrpcClientChannel getChannel(String clientId) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java
index ec14473fd..714d0bf01 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java
@@ -18,13 +18,17 @@ package org.apache.rocketmq.proxy.grpc.v2.channel;
import apache.rocketmq.v2.PrintThreadStackTraceCommand;
import apache.rocketmq.v2.RecoverOrphanedTransactionCommand;
+import apache.rocketmq.v2.Settings;
import apache.rocketmq.v2.TelemetryCommand;
import apache.rocketmq.v2.VerifyMessageCommand;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ComparisonChain;
+import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.TextFormat;
+import com.google.protobuf.util.JsonFormat;
import io.grpc.StatusRuntimeException;
import io.grpc.stub.StreamObserver;
+import io.netty.channel.Channel;
import io.netty.channel.ChannelId;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicReference;
@@ -33,7 +37,14 @@ import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter;
+import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter;
+import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
+import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
+import org.apache.rocketmq.proxy.processor.channel.RemoteChannelConverter;
import org.apache.rocketmq.proxy.service.relay.ProxyChannel;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
@@ -45,24 +56,70 @@ import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequest
import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;
-public class GrpcClientChannel extends ProxyChannel {
+public class GrpcClientChannel extends ProxyChannel implements ChannelExtendAttributeGetter, RemoteChannelConverter {
private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
private final GrpcChannelManager grpcChannelManager;
+ private final GrpcClientSettingsManager grpcClientSettingsManager;
private final AtomicReference<StreamObserver<TelemetryCommand>> telemetryCommandRef = new AtomicReference<>();
private final Object telemetryWriteLock = new Object();
private final String clientId;
- public GrpcClientChannel(ProxyRelayService proxyRelayService, GrpcChannelManager grpcChannelManager,
- ProxyContext ctx, String clientId) {
+ public GrpcClientChannel(ProxyRelayService proxyRelayService, GrpcClientSettingsManager grpcClientSettingsManager,
+ GrpcChannelManager grpcChannelManager, ProxyContext ctx, String clientId) {
super(proxyRelayService, null, new GrpcChannelId(clientId),
ctx.getRemoteAddress(),
ctx.getLocalAddress());
this.grpcChannelManager = grpcChannelManager;
+ this.grpcClientSettingsManager = grpcClientSettingsManager;
this.clientId = clientId;
}
+ @Override
+ public String getChannelExtendAttribute() {
+ Settings settings = this.grpcClientSettingsManager.getRawClientSettings(this.clientId);
+ if (settings == null) {
+ return null;
+ }
+ try {
+ return JsonFormat.printer().print(settings);
+ } catch (InvalidProtocolBufferException e) {
+ log.error("convert settings to json data failed. settings:{}", settings, e);
+ }
+ return null;
+ }
+
+ public static Settings parseChannelExtendAttribute(Channel channel) {
+ if (ChannelHelper.getChannelProtocolType(channel).equals(ChannelProtocolType.GRPC_V2) &&
+ channel instanceof ChannelExtendAttributeGetter) {
+ String attr = ((ChannelExtendAttributeGetter) channel).getChannelExtendAttribute();
+ if (attr == null) {
+ return null;
+ }
+
+ Settings.Builder builder = Settings.newBuilder();
+ try {
+ JsonFormat.parser().merge(attr, builder);
+ return builder.build();
+ } catch (InvalidProtocolBufferException e) {
+ log.error("convert settings json data to settings failed. data:{}", attr, e);
+ return null;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public RemoteChannel toRemoteChannel() {
+ return new RemoteChannel(
+ ConfigurationManager.getProxyConfig().getLocalServeAddr(),
+ this.getRemoteAddress(),
+ this.getLocalAddress(),
+ ChannelProtocolType.GRPC_V2,
+ this.getChannelExtendAttribute());
+ }
+
protected static class GrpcChannelId implements ChannelId {
private final String clientId;
@@ -181,14 +238,6 @@ public class GrpcClientChannel extends ProxyChannel {
return clientId;
}
- public String getRemoteAddress() {
- return remoteAddress;
- }
-
- public String getLocalAddress() {
- return localAddress;
- }
-
public void writeTelemetryCommand(TelemetryCommand command) {
StreamObserver<TelemetryCommand> observer = this.telemetryCommandRef.get();
if (observer == null) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
index 20035f7a1..00cc862a4 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
@@ -32,6 +32,7 @@ import apache.rocketmq.v2.ThreadStackTrace;
import apache.rocketmq.v2.VerifyMessageResult;
import io.grpc.StatusRuntimeException;
import io.grpc.stub.StreamObserver;
+import io.netty.channel.Channel;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -39,6 +40,7 @@ import java.util.concurrent.CompletableFuture;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
import org.apache.rocketmq.broker.client.ProducerChangeListener;
import org.apache.rocketmq.broker.client.ProducerGroupEvent;
@@ -49,6 +51,7 @@ import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity;
import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;
import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;
@@ -209,7 +212,8 @@ public class ClientActivity extends AbstractMessingActivity {
};
}
- protected void processTelemetryException(TelemetryCommand request, Throwable t, StreamObserver<TelemetryCommand> responseObserver) {
+ protected void processTelemetryException(TelemetryCommand request, Throwable t,
+ StreamObserver<TelemetryCommand> responseObserver) {
StatusRuntimeException exception = io.grpc.Status.INTERNAL
.withDescription("process client telemetryCommand failed. " + t.getMessage())
.withCause(t)
@@ -291,7 +295,8 @@ public class ClientActivity extends AbstractMessingActivity {
return channel;
}
- protected GrpcClientChannel registerConsumer(ProxyContext ctx, String consumerGroup, ClientType clientType, List<SubscriptionEntry> subscriptionEntryList, boolean updateSubscription) {
+ protected GrpcClientChannel registerConsumer(ProxyContext ctx, String consumerGroup, ClientType clientType,
+ List<SubscriptionEntry> subscriptionEntryList, boolean updateSubscription) {
String clientId = ctx.getClientID();
LanguageCode languageCode = LanguageCode.valueOf(ctx.getLanguage());
@@ -413,14 +418,67 @@ public class ClientActivity extends AbstractMessingActivity {
@Override
public void handle(ConsumerGroupEvent event, String group, Object... args) {
- if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
- if (args == null || args.length < 1) {
- return;
+ switch (event) {
+ case CLIENT_UNREGISTER:
+ processClientUnregister(group, args);
+ break;
+ case REGISTER:
+ processClientRegister(group, args);
+ break;
+ default:
+ break;
+ }
+ }
+
+ protected void processClientUnregister(String group, Object... args) {
+ if (args == null || args.length < 1) {
+ return;
+ }
+ if (args[0] instanceof ClientChannelInfo) {
+ ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
+ String clientId = clientChannelInfo.getClientId();
+ if (ChannelHelper.isRemote(clientChannelInfo.getChannel())) {
+ grpcClientSettingsManager.computeIfPresent(clientId, orgSettings -> {
+ if (grpcChannelManager.getChannel(clientId) == null) {
+ // if there is no channel connect directly to this proxy
+ return null;
+ }
+ return orgSettings;
+ });
+ } else {
+ grpcChannelManager.removeChannel(clientId);
+ grpcClientSettingsManager.computeIfPresent(clientId, orgSettings -> {
+ ConsumerGroupInfo consumerGroupInfo = messagingProcessor.getConsumerGroupInfo(group);
+ if (consumerGroupInfo == null) {
+ return null;
+ }
+ List<Channel> allChannels = consumerGroupInfo.getAllChannel();
+ if (allChannels == null || allChannels.isEmpty() || allChannels.size() == 1) {
+ // if there is only one channel of this clientId or
+ // there is no channel if this clientId
+ // remove settings of this client
+ return null;
+ }
+ return orgSettings;
+ });
}
- if (args[0] instanceof ClientChannelInfo) {
- ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
- grpcChannelManager.removeChannel(clientChannelInfo.getClientId());
- grpcClientSettingsManager.removeClientSettings(clientChannelInfo.getClientId());
+ }
+ }
+
+ protected void processClientRegister(String group, Object... args) {
+ if (args == null || args.length < 2) {
+ return;
+ }
+ if (args[1] instanceof ClientChannelInfo) {
+ ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[1];
+ Channel channel = clientChannelInfo.getChannel();
+ if (ChannelHelper.isRemote(channel)) {
+ // save settings from channel sync from other proxy
+ Settings settings = GrpcClientChannel.parseChannelExtendAttribute(channel);
+ if (settings == null) {
+ return;
+ }
+ grpcClientSettingsManager.updateClientSettings(clientChannelInfo.getClientId(), settings);
}
}
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java
index 8c3c3a8a4..21c3395ae 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java
@@ -30,6 +30,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
@@ -52,6 +53,10 @@ public class GrpcClientSettingsManager {
this.messagingProcessor = messagingProcessor;
}
+ public Settings getRawClientSettings(String clientId) {
+ return CLIENT_SETTINGS_MAP.get(clientId);
+ }
+
public Settings getClientSettings(ProxyContext ctx) {
String clientId = ctx.getClientID();
Settings settings = CLIENT_SETTINGS_MAP.get(clientId);
@@ -184,6 +189,10 @@ public class GrpcClientSettingsManager {
CLIENT_SETTINGS_MAP.remove(clientId);
}
+ public void computeIfPresent(String clientId, Function<Settings, Settings> function) {
+ CLIENT_SETTINGS_MAP.computeIfPresent(clientId, (clientIdKey, value) -> function.apply(value));
+ }
+
public Settings removeAndGetClientSettings(ProxyContext ctx) {
String clientId = ctx.getClientID();
Settings settings = CLIENT_SETTINGS_MAP.remove(clientId);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ClientProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ClientProcessor.java
index 5408fa066..8fb6eaf7d 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ClientProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ClientProcessor.java
@@ -101,6 +101,11 @@ public class ClientProcessor extends AbstractProcessor {
this.serviceManager.getConsumerManager().unregisterConsumer(consumerGroup, clientChannelInfo, false);
}
+ public void doChannelCloseEvent(String remoteAddr, Channel channel) {
+ this.serviceManager.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);
+ this.serviceManager.getProducerManager().doChannelCloseEvent(remoteAddr, channel);
+ }
+
public void registerConsumerIdsChangeListener(ConsumerIdsChangeListener listener) {
this.serviceManager.getConsumerManager().appendConsumerIdsChangeListener(listener);
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
index 1b7baba0a..66239f0e8 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
@@ -274,6 +274,11 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen
this.clientProcessor.registerConsumerIdsChangeListener(consumerIdsChangeListener);
}
+ @Override
+ public void doChannelCloseEvent(String remoteAddr, Channel channel) {
+ this.clientProcessor.doChannelCloseEvent(remoteAddr, channel);
+ }
+
@Override
public ConsumerGroupInfo getConsumerGroupInfo(String consumerGroup) {
return this.clientProcessor.getConsumerGroupInfo(consumerGroup);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
index 3e8b8084e..3c4e6303f 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
@@ -285,6 +285,8 @@ public interface MessagingProcessor extends StartAndShutdown {
ConsumerIdsChangeListener consumerIdsChangeListener
);
+ void doChannelCloseEvent(String remoteAddr, Channel channel);
+
ConsumerGroupInfo getConsumerGroupInfo(String consumerGroup);
void addTransactionSubscription(
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java
index d435b0c2e..9d000bfe9 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java
@@ -47,6 +47,7 @@ import org.apache.rocketmq.proxy.common.ProxyException;
import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
import org.apache.rocketmq.proxy.common.ReceiptHandleGroup;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
import org.apache.rocketmq.proxy.common.utils.ExceptionUtils;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.config.ProxyConfig;
@@ -105,6 +106,10 @@ public class ReceiptHandleProcessor extends AbstractStartAndShutdown {
}
if (args[0] instanceof ClientChannelInfo) {
ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
+ if (ChannelHelper.isRemote(clientChannelInfo.getChannel())) {
+ // if the channel sync from other proxy is expired, not to clear data of connect to current proxy
+ return;
+ }
clearGroup(new ReceiptHandleGroupKey(clientChannelInfo.getChannel(), group));
}
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelExtendAttributeGetter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelExtendAttributeGetter.java
new file mode 100644
index 000000000..3538a9496
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelExtendAttributeGetter.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.processor.channel;
+
+public interface ChannelExtendAttributeGetter {
+
+ String getChannelExtendAttribute();
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelProtocolType.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelProtocolType.java
new file mode 100644
index 000000000..d2eeb8353
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/ChannelProtocolType.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.processor.channel;
+
+public enum ChannelProtocolType {
+ UNKNOWN("unknown"),
+ GRPC_V2("grpc_v2"),
+ GRPC_V1("grpc_v1"),
+ REMOTING("remoting");
+
+ private final String name;
+
+ ChannelProtocolType(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java
new file mode 100644
index 000000000..5d9c9afcc
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.processor.channel;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelId;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
+
+public class RemoteChannel extends SimpleChannel implements ChannelExtendAttributeGetter {
+ protected final ChannelProtocolType type;
+ protected final String remoteProxyIp;
+ protected volatile String extendAttribute;
+
+ public RemoteChannel(String remoteProxyIp, String remoteAddress, String localAddress, ChannelProtocolType type, String extendAttribute) {
+ super(null,
+ new RemoteChannelId(remoteProxyIp, remoteAddress, localAddress, type),
+ remoteAddress, localAddress);
+ this.type = type;
+ this.remoteProxyIp = remoteProxyIp;
+ this.extendAttribute = extendAttribute;
+ }
+
+ public static class RemoteChannelId implements ChannelId {
+
+ private final String id;
+
+ public RemoteChannelId(String remoteProxyIp, String remoteAddress, String localAddress, ChannelProtocolType type) {
+ this.id = remoteProxyIp + "@" + remoteAddress + "@" + localAddress + "@" + type;
+ }
+
+ @Override
+ public String asShortText() {
+ return this.id;
+ }
+
+ @Override
+ public String asLongText() {
+ return this.id;
+ }
+
+ @Override
+ public int compareTo(ChannelId o) {
+ return this.id.compareTo(o.asLongText());
+ }
+ }
+
+ @Override
+ public boolean isWritable() {
+ return false;
+ }
+
+ public ChannelProtocolType getType() {
+ return type;
+ }
+
+ public String encode() {
+ return RemoteChannelSerializer.toJson(this);
+ }
+
+ public static RemoteChannel decode(String data) {
+ return RemoteChannelSerializer.decodeFromJson(data);
+ }
+
+ public static RemoteChannel create(Channel channel) {
+ if (channel instanceof RemoteChannelConverter) {
+ return ((RemoteChannelConverter) channel).toRemoteChannel();
+ }
+ return null;
+ }
+
+ public String getRemoteProxyIp() {
+ return remoteProxyIp;
+ }
+
+ public void setExtendAttribute(String extendAttribute) {
+ this.extendAttribute = extendAttribute;
+ }
+
+ @Override
+ public String getChannelExtendAttribute() {
+ return this.extendAttribute;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelConverter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelConverter.java
new file mode 100644
index 000000000..9f886e85d
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelConverter.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.processor.channel;
+
+public interface RemoteChannelConverter {
+
+ RemoteChannel toRemoteChannel();
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java
new file mode 100644
index 000000000..8fd216219
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.processor.channel;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+
+public class RemoteChannelSerializer {
+ private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private static final String REMOTE_PROXY_IP_KEY = "remoteProxyIp";
+ private static final String REMOTE_ADDRESS_KEY = "remoteAddress";
+ private static final String LOCAL_ADDRESS_KEY = "localAddress";
+ private static final String TYPE_KEY = "type";
+ private static final String EXTEND_ATTRIBUTE_KEY = "extendAttribute";
+
+ public static String toJson(RemoteChannel remoteChannel) {
+ Map<String, Object> data = new HashMap<>();
+ data.put(REMOTE_PROXY_IP_KEY, remoteChannel.getRemoteProxyIp());
+ data.put(REMOTE_ADDRESS_KEY, remoteChannel.getRemoteAddress());
+ data.put(LOCAL_ADDRESS_KEY, remoteChannel.getLocalAddress());
+ data.put(TYPE_KEY, remoteChannel.getType());
+ data.put(EXTEND_ATTRIBUTE_KEY, remoteChannel.getChannelExtendAttribute());
+ return JSON.toJSONString(data);
+ }
+
+ public static RemoteChannel decodeFromJson(String jsonData) {
+ if (StringUtils.isBlank(jsonData)) {
+ return null;
+ }
+ try {
+ JSONObject jsonObject = JSON.parseObject(jsonData);
+ return new RemoteChannel(
+ jsonObject.getString(REMOTE_PROXY_IP_KEY),
+ jsonObject.getString(REMOTE_ADDRESS_KEY),
+ jsonObject.getString(LOCAL_ADDRESS_KEY),
+ jsonObject.getObject(TYPE_KEY, ChannelProtocolType.class),
+ jsonObject.getObject(EXTEND_ATTRIBUTE_KEY, String.class)
+ );
+ } catch (Throwable t) {
+ log.error("decode remote channel data failed. data:{}", jsonData, t);
+ }
+ return null;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/ClientHousekeepingService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/ClientHousekeepingService.java
new file mode 100644
index 000000000..e213ae855
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/ClientHousekeepingService.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting;
+
+import io.netty.channel.Channel;
+import org.apache.rocketmq.proxy.remoting.activity.ClientManagerActivity;
+import org.apache.rocketmq.remoting.ChannelEventListener;
+
+public class ClientHousekeepingService implements ChannelEventListener {
+
+ private final ClientManagerActivity clientManagerActivity;
+
+ public ClientHousekeepingService(ClientManagerActivity clientManagerActivity) {
+ this.clientManagerActivity = clientManagerActivity;
+ }
+
+ @Override
+ public void onChannelConnect(String remoteAddr, Channel channel) {
+
+ }
+
+ @Override
+ public void onChannelClose(String remoteAddr, Channel channel) {
+ this.clientManagerActivity.doChannelCloseEvent(remoteAddr, channel);
+ }
+
+ @Override
+ public void onChannelException(String remoteAddr, Channel channel) {
+ this.clientManagerActivity.doChannelCloseEvent(remoteAddr, channel);
+ }
+
+ @Override
+ public void onChannelIdle(String remoteAddr, Channel channel) {
+ this.clientManagerActivity.doChannelCloseEvent(remoteAddr, channel);
+ }
+
+}
+
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
new file mode 100644
index 000000000..58b257641
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting;
+
+import io.netty.channel.Channel;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.remoting.RemotingServer;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOutClient {
+
+ private final MessagingProcessor messagingProcessor;
+ private RemotingServer defaultRemotingServer;
+
+ public RemotingProtocolServer(MessagingProcessor messagingProcessor) {
+ this.messagingProcessor = messagingProcessor;
+ }
+
+ protected void init() {
+
+ }
+
+ protected void registerRemotingServer(RemotingServer remotingServer) {
+
+ }
+
+ @Override
+ public void shutdown() throws Exception {
+
+ }
+
+ @Override
+ public void start() throws Exception {
+
+ }
+
+ @Override
+ public CompletableFuture<RemotingCommand> invokeToClient(Channel channel, RemotingCommand request,
+ long timeoutMillis) {
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+ try {
+ this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, responseFuture -> {
+ if (responseFuture.getResponseCommand() == null) {
+ future.completeExceptionally(new MQClientException("response is null after send request to client", responseFuture.getCause()));
+ return;
+ }
+ future.complete(responseFuture.getResponseCommand());
+ });
+ } catch (Throwable t) {
+ future.completeExceptionally(t);
+ }
+ return future;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProxyOutClient.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProxyOutClient.java
new file mode 100644
index 000000000..5a96c41c9
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProxyOutClient.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting;
+
+import io.netty.channel.Channel;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public interface RemotingProxyOutClient {
+
+ CompletableFuture<RemotingCommand> invokeToClient(Channel channel, RemotingCommand request, long timeoutMillis);
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
new file mode 100644
index 000000000..62400e033
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.Set;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
+import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
+import org.apache.rocketmq.broker.client.ProducerChangeListener;
+import org.apache.rocketmq.broker.client.ProducerGroupEvent;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader;
+import org.apache.rocketmq.common.protocol.header.UnregisterClientResponseHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
+import org.apache.rocketmq.common.protocol.heartbeat.ProducerData;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;
+import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class ClientManagerActivity extends AbstractRemotingActivity {
+
+ private final RemotingChannelManager remotingChannelManager;
+
+ public ClientManagerActivity(RequestPipeline requestPipeline, MessagingProcessor messagingProcessor,
+ RemotingChannelManager manager) {
+ super(requestPipeline, messagingProcessor);
+ this.remotingChannelManager = manager;
+ this.init();
+ }
+
+ protected void init() {
+ this.messagingProcessor.registerConsumerListener(new ConsumerIdsChangeListenerImpl());
+ this.messagingProcessor.registerProducerListener(new ProducerChangeListenerImpl());
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ switch (request.getCode()) {
+ case RequestCode.HEART_BEAT:
+ return this.heartBeat(ctx, request, context);
+ case RequestCode.UNREGISTER_CLIENT:
+ return this.unregisterClient(ctx, request, context);
+ case RequestCode.CHECK_CLIENT_CONFIG:
+ return this.checkClientConfig(ctx, request, context);
+ default:
+ break;
+ }
+ return null;
+ }
+
+ protected RemotingCommand heartBeat(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) {
+ HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class);
+ String clientId = heartbeatData.getClientID();
+
+ for (ProducerData data : heartbeatData.getProducerDataSet()) {
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+ this.remotingChannelManager.createProducerChannel(ctx.channel(), data.getGroupName(), clientId),
+ clientId, request.getLanguage(),
+ request.getVersion());
+ messagingProcessor.registerProducer(context, data.getGroupName(), clientChannelInfo);
+ }
+
+ for (ConsumerData data : heartbeatData.getConsumerDataSet()) {
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+ this.remotingChannelManager.createConsumerChannel(ctx.channel(), data.getGroupName(), clientId, data.getSubscriptionDataSet()),
+ clientId, request.getLanguage(),
+ request.getVersion());
+ messagingProcessor.registerConsumer(context, data.getGroupName(), clientChannelInfo, data.getConsumeType(),
+ data.getMessageModel(), data.getConsumeFromWhere(), data.getSubscriptionDataSet(), true);
+ }
+
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark("");
+ return response;
+ }
+
+ protected RemotingCommand unregisterClient(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws RemotingCommandException {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(UnregisterClientResponseHeader.class);
+ final UnregisterClientRequestHeader requestHeader =
+ (UnregisterClientRequestHeader) request.decodeCommandCustomHeader(UnregisterClientRequestHeader.class);
+ final String producerGroup = requestHeader.getProducerGroup();
+ if (producerGroup != null) {
+ RemotingChannel channel = this.remotingChannelManager.removeProducerChannel(producerGroup, ctx.channel());
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+ channel,
+ requestHeader.getClientID(),
+ request.getLanguage(),
+ request.getVersion());
+ this.messagingProcessor.unRegisterProducer(context, producerGroup, clientChannelInfo);
+ }
+ final String consumerGroup = requestHeader.getConsumerGroup();
+ if (consumerGroup != null) {
+ RemotingChannel channel = this.remotingChannelManager.removeConsumerChannel(consumerGroup, ctx.channel());
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+ channel,
+ requestHeader.getClientID(),
+ request.getLanguage(),
+ request.getVersion());
+ this.messagingProcessor.unRegisterConsumer(context, consumerGroup, clientChannelInfo);
+ }
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark("");
+ return response;
+ }
+
+ protected RemotingCommand checkClientConfig(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark("");
+ return response;
+ }
+
+ public void doChannelCloseEvent(String remoteAddr, Channel channel) {
+ Set<RemotingChannel> remotingChannelSet = this.remotingChannelManager.removeChannel(channel);
+ for (RemotingChannel remotingChannel : remotingChannelSet) {
+ this.messagingProcessor.doChannelCloseEvent(remoteAddr, remotingChannel);
+ }
+ }
+
+ protected class ConsumerIdsChangeListenerImpl implements ConsumerIdsChangeListener {
+
+ @Override
+ public void handle(ConsumerGroupEvent event, String group, Object... args) {
+ if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
+ if (args == null || args.length < 1) {
+ return;
+ }
+ if (args[0] instanceof ClientChannelInfo) {
+ ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
+ remotingChannelManager.removeConsumerChannel(group, clientChannelInfo.getChannel());
+ }
+ }
+ }
+
+ @Override
+ public void shutdown() {
+
+ }
+ }
+
+ protected class ProducerChangeListenerImpl implements ProducerChangeListener {
+
+ @Override
+ public void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo) {
+ if (event == ProducerGroupEvent.CLIENT_UNREGISTER) {
+ remotingChannelManager.removeProducerChannel(group, clientChannelInfo.getChannel());
+ }
+ }
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
new file mode 100644
index 000000000..2b2cfca79
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.channel;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFutureListener;
+import java.time.Duration;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult;
+import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
+import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader;
+import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
+import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter;
+import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
+import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
+import org.apache.rocketmq.proxy.processor.channel.RemoteChannelConverter;
+import org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;
+import org.apache.rocketmq.proxy.remoting.common.RemotingConverter;
+import org.apache.rocketmq.proxy.service.relay.ProxyChannel;
+import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;
+import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
+import org.apache.rocketmq.proxy.service.transaction.TransactionData;
+import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class RemotingChannel extends ProxyChannel implements RemoteChannelConverter, ChannelExtendAttributeGetter {
+ private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private static final long DEFAULT_MQ_CLIENT_TIMEOUT = Duration.ofSeconds(3).toMillis();
+ private final String clientId;
+ private final String remoteAddress;
+ private final String localAddress;
+ private final RemotingProxyOutClient remotingProxyOutClient;
+ private final Set<SubscriptionData> subscriptionData;
+
+ public RemotingChannel(RemotingProxyOutClient remotingProxyOutClient, ProxyRelayService proxyRelayService,
+ Channel parent,
+ String clientId, Set<SubscriptionData> subscriptionData) {
+ super(proxyRelayService, parent, parent.id(),
+ RemotingUtil.socketAddress2String(parent.remoteAddress()),
+ RemotingUtil.socketAddress2String(parent.localAddress()));
+ this.remotingProxyOutClient = remotingProxyOutClient;
+ this.clientId = clientId;
+ this.remoteAddress = RemotingUtil.socketAddress2String(parent.remoteAddress());
+ this.localAddress = RemotingUtil.socketAddress2String(parent.localAddress());
+ this.subscriptionData = subscriptionData;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return this.parent().isOpen();
+ }
+
+ @Override
+ public boolean isActive() {
+ return this.parent().isActive();
+ }
+
+ @Override
+ public boolean isWritable() {
+ return this.parent().isWritable();
+ }
+
+ @Override
+ protected CompletableFuture<Void> processOtherMessage(Object msg) {
+ this.parent().writeAndFlush(msg);
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ protected CompletableFuture<Void> processCheckTransaction(CheckTransactionStateRequestHeader header,
+ MessageExt messageExt, TransactionData transactionData,
+ CompletableFuture<ProxyRelayResult<Void>> responseFuture) {
+ CompletableFuture<Void> writeFuture = new CompletableFuture<>();
+ try {
+ CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();
+ requestHeader.setCommitLogOffset(transactionData.getCommitLogOffset());
+ requestHeader.setTranStateTableOffset(transactionData.getTranStateTableOffset());
+ requestHeader.setTransactionId(transactionData.getTransactionId());
+ requestHeader.setMsgId(header.getMsgId());
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader);
+ request.setBody(RemotingConverter.getInstance().convertMsgToBytes(messageExt));
+
+ this.parent().writeAndFlush(request).addListener((ChannelFutureListener) future -> {
+ if (future.isSuccess()) {
+ responseFuture.complete(null);
+ writeFuture.complete(null);
+ } else {
+ Exception e = new RemotingException("write and flush data failed");
+ responseFuture.completeExceptionally(e);
+ writeFuture.completeExceptionally(e);
+ }
+ });
+ } catch (Throwable t) {
+ responseFuture.completeExceptionally(t);
+ writeFuture.completeExceptionally(t);
+ }
+ return writeFuture;
+ }
+
+ @Override
+ protected CompletableFuture<Void> processGetConsumerRunningInfo(RemotingCommand command,
+ GetConsumerRunningInfoRequestHeader header,
+ CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> responseFuture) {
+ CompletableFuture<Void> writeFuture = new CompletableFuture<>();
+ try {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, header);
+ return this.remotingProxyOutClient.invokeToClient(this.parent(), request, DEFAULT_MQ_CLIENT_TIMEOUT)
+ .thenApply(response -> {
+ if (response.getCode() == ResponseCode.SUCCESS) {
+ ConsumerRunningInfo consumerRunningInfo = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class);
+ responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", consumerRunningInfo));
+ return null;
+ }
+ String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark());
+ RuntimeException e = new RuntimeException(errMsg);
+ responseFuture.completeExceptionally(e);
+ throw e;
+ });
+ } catch (Throwable t) {
+ responseFuture.completeExceptionally(t);
+ writeFuture.completeExceptionally(t);
+ }
+ return writeFuture;
+ }
+
+ @Override
+ protected CompletableFuture<Void> processConsumeMessageDirectly(RemotingCommand command,
+ ConsumeMessageDirectlyResultRequestHeader header, MessageExt messageExt,
+ CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> responseFuture) {
+ CompletableFuture<Void> writeFuture = new CompletableFuture<>();
+ try {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, header);
+ request.setBody(RemotingConverter.getInstance().convertMsgToBytes(messageExt));
+
+ return this.remotingProxyOutClient.invokeToClient(this.parent(), request, DEFAULT_MQ_CLIENT_TIMEOUT)
+ .thenApply(response -> {
+ if (response.getCode() == ResponseCode.SUCCESS) {
+ ConsumeMessageDirectlyResult result = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class);
+ responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", result));
+ return null;
+ }
+ String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark());
+ RuntimeException e = new RuntimeException(errMsg);
+ responseFuture.completeExceptionally(e);
+ throw e;
+ });
+ } catch (Throwable t) {
+ responseFuture.completeExceptionally(t);
+ writeFuture.completeExceptionally(t);
+ }
+ return writeFuture;
+ }
+
+ public String getClientId() {
+ return clientId;
+ }
+
+ @Override
+ public String getChannelExtendAttribute() {
+ if (this.subscriptionData == null) {
+ return null;
+ }
+ return JSON.toJSONString(this.subscriptionData);
+ }
+
+ public static Set<SubscriptionData> parseChannelExtendAttribute(Channel channel) {
+ if (ChannelHelper.getChannelProtocolType(channel).equals(ChannelProtocolType.REMOTING) &&
+ channel instanceof ChannelExtendAttributeGetter) {
+ String attr = ((ChannelExtendAttributeGetter) channel).getChannelExtendAttribute();
+ if (attr == null) {
+ return null;
+ }
+
+ try {
+ return JSON.parseObject(attr, new TypeReference<Set<SubscriptionData>>() {
+ });
+ } catch (Exception e) {
+ log.error("convert remoting extend attribute to subscriptionDataSet failed. data:{}", attr, e);
+ return null;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public RemoteChannel toRemoteChannel() {
+ return new RemoteChannel(
+ ConfigurationManager.getProxyConfig().getLocalServeAddr(),
+ this.getRemoteAddress(),
+ this.getLocalAddress(),
+ ChannelProtocolType.REMOTING,
+ this.getChannelExtendAttribute());
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
new file mode 100644
index 000000000..bdc6457e7
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.channel;
+
+import io.netty.channel.Channel;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;
+import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RemotingChannelManager implements StartAndShutdown {
+ protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private final ProxyRelayService proxyRelayService;
+ protected final ConcurrentMap<String /* group */, Map<Channel /* raw channel */, RemotingChannel>> groupChannelMap = new ConcurrentHashMap<>();
+
+ private final RemotingProxyOutClient remotingProxyOutClient;
+
+ public RemotingChannelManager(RemotingProxyOutClient remotingProxyOutClient, ProxyRelayService proxyRelayService) {
+ this.remotingProxyOutClient = remotingProxyOutClient;
+ this.proxyRelayService = proxyRelayService;
+ }
+
+ protected String buildProducerKey(String group) {
+ return buildKey("p", group);
+ }
+
+ protected String buildConsumerKey(String group) {
+ return buildKey("c", group);
+ }
+
+ protected String buildKey(String prefix, String group) {
+ return prefix + group;
+ }
+
+ protected String getGroupFromKey(String key) {
+ return key.substring(1);
+ }
+
+ public RemotingChannel createProducerChannel(Channel channel, String group, String clientId) {
+ return createChannel(channel, buildProducerKey(group), clientId, Collections.emptySet());
+ }
+
+ public RemotingChannel createConsumerChannel(Channel channel, String group, String clientId, Set<SubscriptionData> subscriptionData) {
+ return createChannel(channel, buildConsumerKey(group), clientId, subscriptionData);
+ }
+
+ protected RemotingChannel createChannel(Channel channel, String group, String clientId, Set<SubscriptionData> subscriptionData) {
+ this.groupChannelMap.compute(group, (groupKey, clientIdMap) -> {
+ if (clientIdMap == null) {
+ clientIdMap = new ConcurrentHashMap<>();
+ }
+ clientIdMap.computeIfAbsent(channel, clientIdKey -> new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, subscriptionData));
+ return clientIdMap;
+ });
+ return getChannel(group, channel);
+ }
+
+ public RemotingChannel getConsumerChannel(String group, Channel channel) {
+ return getChannel(buildConsumerKey(group), channel);
+ }
+
+ public RemotingChannel getProducerChannel(String group, Channel channel) {
+ return getChannel(buildProducerKey(group), channel);
+ }
+
+ protected RemotingChannel getChannel(String group, Channel channel) {
+ Map<Channel, RemotingChannel> clientIdChannelMap = this.groupChannelMap.get(group);
+ if (clientIdChannelMap == null) {
+ return null;
+ }
+ return clientIdChannelMap.get(channel);
+ }
+
+ public Set<RemotingChannel> removeChannel(Channel channel) {
+ Set<RemotingChannel> removedChannelSet = new HashSet<>();
+ for (Map.Entry<String, Map<Channel /* raw channel */, RemotingChannel>> entry : groupChannelMap.entrySet()) {
+ Map<Channel /* raw channel */, RemotingChannel> channelMap = entry.getValue();
+
+ RemotingChannel remotingChannel = channelMap.remove(channel);
+ if (remotingChannel != null) {
+ removedChannelSet.add(remotingChannel);
+ }
+ }
+ return removedChannelSet;
+ }
+
+ public RemotingChannel removeProducerChannel(String group, Channel channel) {
+ return removeChannel(buildProducerKey(group), channel);
+ }
+
+ public RemotingChannel removeConsumerChannel(String group, Channel channel) {
+ return removeChannel(buildConsumerKey(group), channel);
+ }
+
+ protected RemotingChannel removeChannel(String group, Channel channel) {
+ AtomicReference<RemotingChannel> channelRef = new AtomicReference<>();
+
+ this.groupChannelMap.computeIfPresent(group, (groupKey, channelMap) -> {
+ channelRef.set(channelMap.remove(channel));
+ if (channelMap.isEmpty()) {
+ return null;
+ }
+ return channelMap;
+ });
+ return channelRef.get();
+ }
+
+ @Override
+ public void shutdown() throws Exception {
+
+ }
+
+ @Override
+ public void start() throws Exception {
+
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java
new file mode 100644
index 000000000..55af1ff19
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.common;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+
+public class RemotingConverter {
+ private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+
+ protected static final Object INSTANCE_CREATE_LOCK = new Object();
+ protected static volatile RemotingConverter instance;
+
+ public static RemotingConverter getInstance() {
+ if (instance == null) {
+ synchronized (INSTANCE_CREATE_LOCK) {
+ if (instance == null) {
+ instance = new RemotingConverter();
+ }
+ }
+ }
+ return instance;
+ }
+
+ public byte[] convertMsgToBytes(List<MessageExt> msgList) {
+ // set response body
+ byte[][] msgBufferList = new byte[msgList.size()][];
+ int bodyTotalSize = 0;
+ for (int i = 0; i < msgList.size(); i++) {
+ try {
+ msgBufferList[i] = convertMsgToBytes(msgList.get(i));
+ bodyTotalSize += msgBufferList[i].length;
+ } catch (Exception e) {
+ log.error("messageToByteBuffer UnsupportedEncodingException", e);
+ }
+ }
+
+ ByteBuffer body = ByteBuffer.allocate(bodyTotalSize);
+ for (byte[] bb : msgBufferList) {
+ body.put(bb);
+ }
+
+ return body.array();
+ }
+
+ public byte[] convertMsgToBytes(final MessageExt msg) throws Exception {
+ // change to 0 for recalculate storeSize
+ msg.setStoreSize(0);
+ if (msg.getTopic().length() > Byte.MAX_VALUE) {
+ log.warn("Topic length is too long, topic: {}", msg.getTopic());
+ }
+ return MessageDecoder.encode(msg, false);
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
index ac1ff6a88..24b27aaa2 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
@@ -32,6 +32,9 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.service.admin.AdminService;
+import org.apache.rocketmq.proxy.service.admin.DefaultAdminService;
+import org.apache.rocketmq.proxy.service.client.ClusterConsumerManager;
import org.apache.rocketmq.proxy.service.message.ClusterMessageService;
import org.apache.rocketmq.proxy.service.message.MessageService;
import org.apache.rocketmq.proxy.service.metadata.ClusterMetadataService;
@@ -52,11 +55,12 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
protected ClusterTransactionService clusterTransactionService;
protected ProducerManager producerManager;
- protected ConsumerManager consumerManager;
+ protected ClusterConsumerManager consumerManager;
protected TopicRouteService topicRouteService;
protected MessageService messageService;
protected ProxyRelayService proxyRelayService;
protected ClusterMetadataService metadataService;
+ protected AdminService adminService;
protected ScheduledExecutorService scheduledExecutorService;
protected MQClientAPIFactory messagingClientAPIFactory;
@@ -67,7 +71,7 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
this.scheduledExecutorService = Executors.newScheduledThreadPool(3);
this.producerManager = new ProducerManager();
- this.consumerManager = new ConsumerManager(new ConsumerIdsChangeListenerImpl(), proxyConfig.getChannelExpiredTimeout());
+ this.consumerManager = new ClusterConsumerManager(this.topicRouteService, this.adminService, this.operationClientAPIFactory, new ConsumerIdsChangeListenerImpl(), proxyConfig.getChannelExpiredTimeout());
this.messagingClientAPIFactory = new MQClientAPIFactory(
"ClusterMQClient_",
@@ -76,7 +80,7 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
rpcHook,
scheduledExecutorService);
this.operationClientAPIFactory = new MQClientAPIFactory(
- "TopicRouteServiceClient_",
+ "OperationClient_",
1,
new DoNothingClientRemotingProcessor(null),
rpcHook,
@@ -95,6 +99,7 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
this.transactionClientAPIFactory);
this.proxyRelayService = new ClusterProxyRelayService(this.clusterTransactionService);
this.metadataService = new ClusterMetadataService(topicRouteService, operationClientAPIFactory);
+ this.adminService = new DefaultAdminService(this.operationClientAPIFactory);
this.init();
}
@@ -118,6 +123,7 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
this.appendStartAndShutdown(this.topicRouteService);
this.appendStartAndShutdown(this.clusterTransactionService);
this.appendStartAndShutdown(this.metadataService);
+ this.appendStartAndShutdown(this.consumerManager);
}
@Override
@@ -155,6 +161,11 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
return this.metadataService;
}
+ @Override
+ public AdminService getAdminService() {
+ return this.adminService;
+ }
+
protected static class ConsumerIdsChangeListenerImpl implements ConsumerIdsChangeListener {
@Override
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java
index 6afc86c57..4f829caa6 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java
@@ -25,6 +25,8 @@ import org.apache.rocketmq.broker.client.ProducerManager;
import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.service.admin.AdminService;
+import org.apache.rocketmq.proxy.service.admin.DefaultAdminService;
import org.apache.rocketmq.proxy.service.channel.ChannelManager;
import org.apache.rocketmq.proxy.service.message.LocalMessageService;
import org.apache.rocketmq.proxy.service.message.MessageService;
@@ -48,6 +50,7 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser
private final TransactionService transactionService;
private final ProxyRelayService proxyRelayService;
private final MetadataService metadataService;
+ private final AdminService adminService;
private final MQClientAPIFactory mqClientAPIFactory;
private final ChannelManager channelManager;
@@ -60,7 +63,7 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser
this.channelManager = new ChannelManager();
this.messageService = new LocalMessageService(brokerController, channelManager, rpcHook);
this.mqClientAPIFactory = new MQClientAPIFactory(
- "TopicRouteServiceClient_",
+ "LocalMQClient_",
1,
new DoNothingClientRemotingProcessor(null),
rpcHook,
@@ -70,6 +73,7 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser
this.transactionService = new LocalTransactionService(brokerController.getBrokerConfig());
this.proxyRelayService = new LocalProxyRelayService(brokerController, this.transactionService);
this.metadataService = new LocalMetadataService(brokerController);
+ this.adminService = new DefaultAdminService(this.mqClientAPIFactory);
this.init();
}
@@ -114,6 +118,11 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser
return this.metadataService;
}
+ @Override
+ public AdminService getAdminService() {
+ return this.adminService;
+ }
+
private class LocalServiceManagerStartAndShutdown implements StartAndShutdown {
@Override
public void start() throws Exception {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java
index 563b56715..ce84832ca 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java
@@ -16,9 +16,10 @@
*/
package org.apache.rocketmq.proxy.service;
-import org.apache.rocketmq.broker.client.ConsumerManager;
-import org.apache.rocketmq.broker.client.ProducerManager;
+import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
+import org.apache.rocketmq.broker.client.ProducerManagerInterface;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.service.admin.AdminService;
import org.apache.rocketmq.proxy.service.message.MessageService;
import org.apache.rocketmq.proxy.service.metadata.MetadataService;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
@@ -30,13 +31,15 @@ public interface ServiceManager extends StartAndShutdown {
TopicRouteService getTopicRouteService();
- ProducerManager getProducerManager();
+ ProducerManagerInterface getProducerManager();
- ConsumerManager getConsumerManager();
+ ConsumerManagerInterface getConsumerManager();
TransactionService getTransactionService();
ProxyRelayService getProxyRelayService();
MetadataService getMetadataService();
+
+ AdminService getAdminService();
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java
new file mode 100644
index 000000000..d98d17ff7
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.admin;
+
+import java.util.List;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+
+public interface AdminService {
+
+ boolean topicExist(String topic);
+
+ boolean createTopicOnTopicBrokerIfNotExist(String createTopic, String sampleTopic, int wQueueNum,
+ int rQueueNum, boolean examineTopic, int retryCheckCount);
+
+ boolean createTopicOnBroker(String topic, int wQueueNum, int rQueueNum, List<BrokerData> curBrokerDataList,
+ List<BrokerData> sampleBrokerDataList, boolean examineTopic, int retryCheckCount) throws Exception;
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
new file mode 100644
index 000000000..4f3b407d6
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.admin;
+
+import java.time.Duration;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIExt;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
+import org.apache.rocketmq.proxy.service.route.TopicRouteHelper;
+
+public class DefaultAdminService implements AdminService {
+ private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private final MQClientAPIFactory mqClientAPIFactory;
+
+ public DefaultAdminService(MQClientAPIFactory mqClientAPIFactory) {
+ this.mqClientAPIFactory = mqClientAPIFactory;
+ }
+
+ @Override
+ public boolean topicExist(String topic) {
+ boolean topicExist;
+ TopicRouteData topicRouteData;
+ try {
+ topicRouteData = this.getTopicRouteDataDirectlyFromNameServer(topic);
+ topicExist = topicRouteData != null;
+ } catch (Throwable e) {
+ topicExist = false;
+ }
+
+ return topicExist;
+ }
+
+ @Override
+ public boolean createTopicOnTopicBrokerIfNotExist(String createTopic, String sampleTopic, int wQueueNum,
+ int rQueueNum, boolean examineTopic, int retryCheckCount) {
+ TopicRouteData curTopicRouteData = new TopicRouteData();
+ try {
+ curTopicRouteData = this.getTopicRouteDataDirectlyFromNameServer(createTopic);
+ } catch (Exception e) {
+ if (!TopicRouteHelper.isTopicNotExistError(e)) {
+ log.error("get cur topic route {} failed.", createTopic, e);
+ return false;
+ }
+ }
+
+ TopicRouteData sampleTopicRouteData = null;
+ try {
+ sampleTopicRouteData = this.getTopicRouteDataDirectlyFromNameServer(sampleTopic);
+ } catch (Exception e) {
+ log.error("create topic {} failed.", createTopic, e);
+ return false;
+ }
+
+ if (sampleTopicRouteData == null || sampleTopicRouteData.getBrokerDatas().isEmpty()) {
+ return false;
+ }
+
+ try {
+ return this.createTopicOnBroker(createTopic, wQueueNum, rQueueNum, curTopicRouteData.getBrokerDatas(),
+ sampleTopicRouteData.getBrokerDatas(), examineTopic, retryCheckCount);
+ } catch (Exception e) {
+ log.error("create topic {} failed.", createTopic, e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean createTopicOnBroker(String topic, int wQueueNum, int rQueueNum, List<BrokerData> curBrokerDataList,
+ List<BrokerData> sampleBrokerDataList, boolean examineTopic, int retryCheckCount) throws Exception {
+ Set<String> curBrokerAddr = new HashSet<>();
+ if (curBrokerDataList != null) {
+ for (BrokerData brokerData : curBrokerDataList) {
+ curBrokerAddr.add(brokerData.getBrokerAddrs().get(MixAll.MASTER_ID));
+ }
+ }
+
+ TopicConfig topicConfig = new TopicConfig();
+ topicConfig.setTopicName(topic);
+ topicConfig.setWriteQueueNums(wQueueNum);
+ topicConfig.setReadQueueNums(rQueueNum);
+ topicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);
+
+ for (BrokerData brokerData : sampleBrokerDataList) {
+ String addr = brokerData.getBrokerAddrs() == null ? null : brokerData.getBrokerAddrs().get(MixAll.MASTER_ID);
+ if (addr == null) {
+ continue;
+ }
+ if (curBrokerAddr.contains(addr)) {
+ continue;
+ }
+
+ this.getClient().createTopic(addr, TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, topicConfig, Duration.ofSeconds(3).toMillis());
+ }
+
+ if (examineTopic) {
+ // examine topic exist.
+ int count = retryCheckCount;
+ while (count-- > 0) {
+ if (this.topicExist(topic)) {
+ return true;
+ }
+ }
+ } else {
+ return true;
+ }
+ return false;
+ }
+
+ protected TopicRouteData getTopicRouteDataDirectlyFromNameServer(String topic) throws Exception {
+ return this.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis());
+ }
+
+ protected MQClientAPIExt getClient() {
+ return this.mqClientAPIFactory.getClient();
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java
index 04ad5e269..65c1fd406 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java
@@ -196,6 +196,14 @@ public class SimpleChannel extends AbstractChannel {
}
+ public String getRemoteAddress() {
+ return remoteAddress;
+ }
+
+ public String getLocalAddress() {
+ return localAddress;
+ }
+
public ChannelHandlerContext getChannelHandlerContext() {
return channelHandlerContext;
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
new file mode 100644
index 000000000..3bb65b03e
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.client;
+
+import java.util.Set;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
+import org.apache.rocketmq.broker.client.ConsumerManager;
+import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.service.admin.AdminService;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
+import org.apache.rocketmq.proxy.service.route.TopicRouteService;
+import org.apache.rocketmq.proxy.service.sysmessage.HeartbeatSyncer;
+
+public class ClusterConsumerManager extends ConsumerManager implements ConsumerManagerInterface, StartAndShutdown {
+
+ protected HeartbeatSyncer heartbeatSyncer;
+
+ public ClusterConsumerManager(TopicRouteService topicRouteService, AdminService adminService,
+ MQClientAPIFactory mqClientAPIFactory, ConsumerIdsChangeListener consumerIdsChangeListener, long channelExpiredTimeout) {
+ super(consumerIdsChangeListener, channelExpiredTimeout);
+ this.heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, this, mqClientAPIFactory);
+ }
+
+ @Override
+ public boolean registerConsumer(String group, ClientChannelInfo clientChannelInfo,
+ ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
+ Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription) {
+ this.heartbeatSyncer.onConsumerRegister(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList);
+ return super.registerConsumer(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList,
+ isNotifyConsumerIdsChangedEnable, updateSubscription);
+ }
+
+ @Override
+ public void unregisterConsumer(String group, ClientChannelInfo clientChannelInfo,
+ boolean isNotifyConsumerIdsChangedEnable) {
+ this.heartbeatSyncer.onConsumerUnRegister(group, clientChannelInfo);
+ super.unregisterConsumer(group, clientChannelInfo, isNotifyConsumerIdsChangedEnable);
+ }
+
+ @Override
+ public void shutdown() throws Exception {
+ this.heartbeatSyncer.shutdown();
+ }
+
+ @Override
+ public void start() throws Exception {
+ this.heartbeatSyncer.start();
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
new file mode 100644
index 000000000..72593525e
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.sysmessage;
+
+import com.alibaba.fastjson.JSON;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.proxy.common.ProxyException;
+import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
+import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.service.admin.AdminService;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
+import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
+import org.apache.rocketmq.proxy.service.route.TopicRouteService;
+import org.apache.rocketmq.remoting.RPCHook;
+
+public abstract class AbstractSystemMessageSyncer implements StartAndShutdown, MessageListenerConcurrently {
+ protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ protected final TopicRouteService topicRouteService;
+ protected final AdminService adminService;
+ protected final String systemResourceName;
+ protected final MQClientAPIFactory mqClientAPIFactory;
+ protected DefaultMQPushConsumer defaultMQPushConsumer;
+
+ public AbstractSystemMessageSyncer(TopicRouteService topicRouteService, AdminService adminService, MQClientAPIFactory mqClientAPIFactory) {
+ this.topicRouteService = topicRouteService;
+ this.adminService = adminService;
+ this.mqClientAPIFactory = mqClientAPIFactory;
+
+ this.systemResourceName = this.getSystemResourceName();
+ }
+
+ protected String getSystemResourceName() {
+ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
+ return TopicValidator.SYSTEM_TOPIC_PREFIX + "proxy_" + this.getClass().getSimpleName() + "_" + proxyConfig.getProxyClusterName();
+ }
+
+ protected String getSystemMessageProducerId() {
+ return "PID_" + this.systemResourceName;
+ }
+
+ protected String getSystemMessageConsumerId() {
+ return "CID_" + this.systemResourceName;
+ }
+
+ protected String getBroadcastTopicName() {
+ return this.systemResourceName;
+ }
+
+ protected String getSubTag() {
+ return "*";
+ }
+
+ protected String getBroadcastTopicClusterName() {
+ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
+ return proxyConfig.getSystemTopicClusterName();
+ }
+
+ protected int getBroadcastTopicQueueNum() {
+ return 1;
+ }
+
+ protected RPCHook getRpcHook() {
+ return null;
+ }
+
+ protected void sendSystemMessage(Object data) {
+ String targetTopic = this.getBroadcastTopicName();
+ try {
+ Message message = new Message(
+ targetTopic,
+ JSON.toJSONString(data).getBytes(StandardCharsets.UTF_8)
+ );
+
+ AddressableMessageQueue messageQueue = this.topicRouteService.getAllMessageQueueView(targetTopic)
+ .getWriteSelector().selectOne(true);
+ this.mqClientAPIFactory.getClient().sendMessageAsync(
+ messageQueue.getBrokerAddr(),
+ messageQueue.getBrokerName(),
+ message,
+ buildSendMessageRequestHeader(message, this.getSystemMessageProducerId(), messageQueue.getQueueId()),
+ Duration.ofSeconds(3).toMillis()
+ ).whenCompleteAsync((result, throwable) -> {
+ if (throwable != null) {
+ log.error("send system message failed. data: {}, topic: {}", data, getBroadcastTopicName(), throwable);
+ return;
+ }
+ if (SendStatus.SEND_OK != result.getSendStatus()) {
+ log.error("send system message failed. data: {}, topic: {}, sendResult:{}", data, getBroadcastTopicName(), result);
+ }
+ });
+ } catch (Throwable t) {
+ log.error("send system message failed. data: {}, topic: {}", data, targetTopic, t);
+ }
+ }
+
+ protected SendMessageRequestHeader buildSendMessageRequestHeader(Message message,
+ String producerGroup, int queueId) {
+ SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+
+ requestHeader.setProducerGroup(producerGroup);
+ requestHeader.setTopic(message.getTopic());
+ requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);
+ requestHeader.setDefaultTopicQueueNums(0);
+ requestHeader.setQueueId(queueId);
+ requestHeader.setSysFlag(0);
+ requestHeader.setBornTimestamp(System.currentTimeMillis());
+ requestHeader.setFlag(message.getFlag());
+ requestHeader.setProperties(MessageDecoder.messageProperties2String(message.getProperties()));
+ requestHeader.setReconsumeTimes(0);
+ requestHeader.setBatch(false);
+ return requestHeader;
+ }
+
+ @Override
+ public void start() throws Exception {
+ this.createSysTopic();
+ RPCHook rpcHook = this.getRpcHook();
+ this.defaultMQPushConsumer = new DefaultMQPushConsumer(null, this.getSystemMessageConsumerId(), rpcHook);
+
+ this.defaultMQPushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
+ this.defaultMQPushConsumer.setMessageModel(MessageModel.BROADCASTING);
+ try {
+ this.defaultMQPushConsumer.subscribe(this.getBroadcastTopicName(), this.getSubTag());
+ } catch (MQClientException e) {
+ throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "subscribe to broadcast topic " + this.getBroadcastTopicName() + " failed. " + e.getMessage());
+ }
+ this.defaultMQPushConsumer.registerMessageListener(this);
+ this.defaultMQPushConsumer.start();
+ }
+
+ protected void createSysTopic() {
+ if (this.adminService.topicExist(this.getBroadcastTopicName())) {
+ return;
+ }
+
+ String clusterName = this.getBroadcastTopicClusterName();
+ if (StringUtils.isEmpty(clusterName)) {
+ throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "system topic cluster cannot be empty");
+ }
+
+ boolean createSuccess = this.adminService.createTopicOnTopicBrokerIfNotExist(
+ this.getBroadcastTopicName(),
+ clusterName,
+ this.getBroadcastTopicQueueNum(),
+ this.getBroadcastTopicQueueNum(),
+ true,
+ 3
+ );
+ if (!createSuccess) {
+ throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "create system broadcast topic " + this.getBroadcastTopicName() + " failed on cluster " + clusterName);
+ }
+ }
+
+ @Override
+ public void shutdown() throws Exception {
+ this.defaultMQPushConsumer.shutdown();
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
new file mode 100644
index 000000000..ce3403766
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.sysmessage;
+
+import com.alibaba.fastjson.JSON;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
+import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
+import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
+import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
+import org.apache.rocketmq.proxy.service.admin.AdminService;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
+import org.apache.rocketmq.proxy.service.route.TopicRouteService;
+
+public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
+
+ protected ThreadPoolExecutor threadPoolExecutor;
+ protected ConsumerManagerInterface consumerManager;
+ protected final Map<String /* channelId as longText */, RemoteChannel> remoteChannelMap = new ConcurrentHashMap<>();
+
+ public HeartbeatSyncer(TopicRouteService topicRouteService, AdminService adminService,
+ ConsumerManagerInterface consumerManager, MQClientAPIFactory mqClientAPIFactory) {
+ super(topicRouteService, adminService, mqClientAPIFactory);
+ this.consumerManager = consumerManager;
+ this.init();
+ }
+
+ protected void init() {
+ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
+ this.threadPoolExecutor = ThreadPoolMonitor.createAndMonitor(
+ proxyConfig.getHeartbeatSyncerThreadPoolNums(),
+ proxyConfig.getHeartbeatSyncerThreadPoolNums(),
+ 1,
+ TimeUnit.MINUTES,
+ "HeartbeatSyncer",
+ proxyConfig.getHeartbeatSyncerThreadPoolQueueCapacity()
+ );
+ this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() {
+ @Override
+ public void handle(ConsumerGroupEvent event, String s, Object... args) {
+ if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
+ if (args == null || args.length < 1) {
+ return;
+ }
+ if (args[0] instanceof ClientChannelInfo) {
+ ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
+ remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText());
+ }
+ }
+ }
+
+ @Override
+ public void shutdown() {
+
+ }
+ });
+ }
+
+ @Override
+ public void shutdown() throws Exception {
+ this.threadPoolExecutor.shutdown();
+ super.shutdown();
+ }
+
+ public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo,
+ ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
+ Set<SubscriptionData> subList) {
+ if (clientChannelInfo == null || ChannelHelper.isRemote(clientChannelInfo.getChannel())) {
+ return;
+ }
+ try {
+ this.threadPoolExecutor.submit(() -> {
+ try {
+ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
+ RemoteChannel remoteChannel = RemoteChannel.create(clientChannelInfo.getChannel());
+ if (remoteChannel == null) {
+ return;
+ }
+ HeartbeatSyncerData data = new HeartbeatSyncerData(
+ HeartbeatType.REGISTER,
+ clientChannelInfo.getClientId(),
+ clientChannelInfo.getLanguage(),
+ clientChannelInfo.getVersion(),
+ consumerGroup,
+ consumeType,
+ messageModel,
+ consumeFromWhere,
+ proxyConfig.getLocalServeAddr(),
+ remoteChannel.encode(),
+ remoteChannel.getChannelExtendAttribute()
+ );
+ data.setSubscriptionDataSet(subList);
+
+ this.sendSystemMessage(data);
+ } catch (Throwable t) {
+ log.error("heartbeat register broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}, messageModel:{}, consumeFromWhere:{}, subList:{}",
+ consumerGroup, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList, t);
+ }
+ });
+ } catch (Throwable t) {
+ log.error("heartbeat submit register broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}, messageModel:{}, consumeFromWhere:{}, subList:{}",
+ consumerGroup, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList, t);
+ }
+ }
+
+ public void onConsumerUnRegister(String consumerGroup, ClientChannelInfo clientChannelInfo) {
+ if (clientChannelInfo == null || ChannelHelper.isRemote(clientChannelInfo.getChannel())) {
+ return;
+ }
+ try {
+ this.threadPoolExecutor.submit(() -> {
+ try {
+ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
+ RemoteChannel remoteChannel = RemoteChannel.create(clientChannelInfo.getChannel());
+ if (remoteChannel == null) {
+ return;
+ }
+ HeartbeatSyncerData data = new HeartbeatSyncerData(
+ HeartbeatType.UNREGISTER,
+ clientChannelInfo.getClientId(),
+ clientChannelInfo.getLanguage(),
+ clientChannelInfo.getVersion(),
+ consumerGroup,
+ null,
+ null,
+ null,
+ proxyConfig.getLocalServeAddr(),
+ remoteChannel.encode(),
+ remoteChannel.getChannelExtendAttribute()
+ );
+
+ this.sendSystemMessage(data);
+ } catch (Throwable t) {
+ log.error("heartbeat unregister broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}",
+ consumerGroup, clientChannelInfo, t);
+ }
+ });
+ } catch (Throwable t) {
+ log.error("heartbeat submit unregister broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}",
+ consumerGroup, clientChannelInfo, t);
+ }
+ }
+
+ @Override
+ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+ if (msgs == null || msgs.isEmpty()) {
+ return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+ }
+
+ for (MessageExt msg : msgs) {
+ try {
+ HeartbeatSyncerData data = JSON.parseObject(new String(msg.getBody(), StandardCharsets.UTF_8), HeartbeatSyncerData.class);
+ if (data.getConnectProxyIp().equals(ConfigurationManager.getProxyConfig().getLocalServeAddr())) {
+ continue;
+ }
+
+ RemoteChannel channel = RemoteChannel.decode(data.getChannelData());
+ RemoteChannel finalChannel = channel;
+ channel = remoteChannelMap.computeIfAbsent(channel.id().asLongText(), key -> finalChannel);
+ channel.setExtendAttribute(data.getChannelExtendAttribute());
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+ channel,
+ data.getClientId(),
+ data.getLanguage(),
+ data.getVersion()
+ );
+ if (data.getHeartbeatType().equals(HeartbeatType.REGISTER)) {
+ this.consumerManager.registerConsumer(
+ data.getGroup(),
+ clientChannelInfo,
+ data.getConsumeType(),
+ data.getMessageModel(),
+ data.getConsumeFromWhere(),
+ data.getSubscriptionDataSet(),
+ false
+ );
+ } else {
+ this.consumerManager.unregisterConsumer(
+ data.getGroup(),
+ clientChannelInfo,
+ false
+ );
+ }
+ } catch (Throwable t) {
+ log.error("heartbeat consume message failed. msg:{}, data:{}", msg, new String(msg.getBody(), StandardCharsets.UTF_8), t);
+ }
+ }
+
+ return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
new file mode 100644
index 000000000..20fee7aac
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.sysmessage;
+
+import java.util.Set;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
+
+public class HeartbeatSyncerData {
+ private HeartbeatType heartbeatType;
+ private String clientId;
+ private LanguageCode language;
+ private int version;
+ private long lastUpdateTimestamp = System.currentTimeMillis();
+ private Set<SubscriptionData> subscriptionDataSet;
+ private String group;
+ private ConsumeType consumeType;
+ private MessageModel messageModel;
+ private ConsumeFromWhere consumeFromWhere;
+ private String connectProxyIp;
+ private String channelData;
+ private String channelExtendAttribute;
+
+ public HeartbeatSyncerData(HeartbeatType heartbeatType, String clientId,
+ LanguageCode language, int version, String group,
+ ConsumeType consumeType, MessageModel messageModel,
+ ConsumeFromWhere consumeFromWhere, String connectProxyIp,
+ String channelData, String channelExtendAttribute) {
+ this.heartbeatType = heartbeatType;
+ this.clientId = clientId;
+ this.language = language;
+ this.version = version;
+ this.group = group;
+ this.consumeType = consumeType;
+ this.messageModel = messageModel;
+ this.consumeFromWhere = consumeFromWhere;
+ this.connectProxyIp = connectProxyIp;
+ this.channelData = channelData;
+ this.channelExtendAttribute = channelExtendAttribute;
+ }
+
+ public HeartbeatType getHeartbeatType() {
+ return heartbeatType;
+ }
+
+ public void setHeartbeatType(HeartbeatType heartbeatType) {
+ this.heartbeatType = heartbeatType;
+ }
+
+ public String getClientId() {
+ return clientId;
+ }
+
+ public void setClientId(String clientId) {
+ this.clientId = clientId;
+ }
+
+ public LanguageCode getLanguage() {
+ return language;
+ }
+
+ public void setLanguage(LanguageCode language) {
+ this.language = language;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ public void setVersion(int version) {
+ this.version = version;
+ }
+
+ public long getLastUpdateTimestamp() {
+ return lastUpdateTimestamp;
+ }
+
+ public void setLastUpdateTimestamp(long lastUpdateTimestamp) {
+ this.lastUpdateTimestamp = lastUpdateTimestamp;
+ }
+
+ public Set<SubscriptionData> getSubscriptionDataSet() {
+ return subscriptionDataSet;
+ }
+
+ public void setSubscriptionDataSet(
+ Set<SubscriptionData> subscriptionDataSet) {
+ this.subscriptionDataSet = subscriptionDataSet;
+ }
+
+ public String getGroup() {
+ return group;
+ }
+
+ public void setGroup(String group) {
+ this.group = group;
+ }
+
+ public ConsumeType getConsumeType() {
+ return consumeType;
+ }
+
+ public void setConsumeType(ConsumeType consumeType) {
+ this.consumeType = consumeType;
+ }
+
+ public MessageModel getMessageModel() {
+ return messageModel;
+ }
+
+ public void setMessageModel(MessageModel messageModel) {
+ this.messageModel = messageModel;
+ }
+
+ public ConsumeFromWhere getConsumeFromWhere() {
+ return consumeFromWhere;
+ }
+
+ public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) {
+ this.consumeFromWhere = consumeFromWhere;
+ }
+
+ public String getConnectProxyIp() {
+ return connectProxyIp;
+ }
+
+ public void setConnectProxyIp(String connectProxyIp) {
+ this.connectProxyIp = connectProxyIp;
+ }
+
+ public String getChannelData() {
+ return channelData;
+ }
+
+ public void setChannelData(String channelData) {
+ this.channelData = channelData;
+ }
+
+ public String getChannelExtendAttribute() {
+ return channelExtendAttribute;
+ }
+
+ public void setChannelExtendAttribute(String channelExtendAttribute) {
+ this.channelExtendAttribute = channelExtendAttribute;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatType.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatType.java
new file mode 100644
index 000000000..8f0801f54
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatType.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.sysmessage;
+
+public enum HeartbeatType {
+ REGISTER,
+ UNREGISTER;
+}
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java
index 3a50d842f..eb90b9205 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java
@@ -36,6 +36,7 @@ import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.utils.NetworkUtil;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;
+import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
@@ -64,6 +65,8 @@ public class ProxyClientRemotingProcessorTest {
@Mock
private ProducerManager producerManager;
@Mock
+ private GrpcClientSettingsManager grpcClientSettingsManager;
+ @Mock
private ProxyRelayService proxyRelayService;
@Test
@@ -74,7 +77,7 @@ public class ProxyClientRemotingProcessorTest {
new TransactionData("brokerName", 0, 0, "id", System.currentTimeMillis(), 3000),
proxyRelayResultFuture));
- GrpcClientChannel grpcClientChannel = new GrpcClientChannel(proxyRelayService, null,
+ GrpcClientChannel grpcClientChannel = new GrpcClientChannel(proxyRelayService, grpcClientSettingsManager, null,
ProxyContext.create().setRemoteAddress("127.0.0.1:8888").setLocalAddress("127.0.0.1:10911"), "clientId");
when(producerManager.getAvailableChannel(anyString()))
.thenReturn(grpcClientChannel);
[rocketmq] 09/26: [ISSUE #5406] support transaction message for remoting proxy
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 7e916e3db8421acad01b9fce4743b92d51a4b11d
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Wed Nov 9 16:58:01 2022 +0800
[ISSUE #5406] support transaction message for remoting proxy
---
.../proxy/remoting/RemotingProtocolServer.java | 5 ++
.../remoting/activity/SendMessageActivity.java | 8 +--
.../remoting/activity/TransactionActivity.java | 68 ++++++++++++++++++++++
3 files changed, 74 insertions(+), 7 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
index 91c4422d2..d0137b2b4 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -44,6 +44,7 @@ import org.apache.rocketmq.proxy.remoting.activity.GetTopicRouteActivity;
import org.apache.rocketmq.proxy.remoting.activity.PopMessageActivity;
import org.apache.rocketmq.proxy.remoting.activity.PullMessageActivity;
import org.apache.rocketmq.proxy.remoting.activity.SendMessageActivity;
+import org.apache.rocketmq.proxy.remoting.activity.TransactionActivity;
import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;
import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
import org.apache.rocketmq.remoting.ChannelEventListener;
@@ -67,6 +68,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
protected final ClientManagerActivity clientManagerActivity;
protected final ConsumerManagerActivity consumerManagerActivity;
protected final SendMessageActivity sendMessageActivity;
+ protected final TransactionActivity transactionActivity;
protected final PullMessageActivity pullMessageActivity;
protected final PopMessageActivity popMessageActivity;
protected final AckMessageActivity ackMessageActivity;
@@ -88,6 +90,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
this.clientManagerActivity = new ClientManagerActivity(pipeline, messagingProcessor, remotingChannelManager);
this.consumerManagerActivity = new ConsumerManagerActivity(pipeline, messagingProcessor);
this.sendMessageActivity = new SendMessageActivity(pipeline, messagingProcessor);
+ this.transactionActivity = new TransactionActivity(pipeline, messagingProcessor);
this.pullMessageActivity = new PullMessageActivity(pipeline, messagingProcessor);
this.popMessageActivity = new PopMessageActivity(pipeline, messagingProcessor);
this.ackMessageActivity = new AckMessageActivity(pipeline, messagingProcessor);
@@ -184,6 +187,8 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageActivity, this.sendMessageExecutor);
remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageActivity, sendMessageExecutor);
+ remotingServer.registerProcessor(RequestCode.END_TRANSACTION, transactionActivity, sendMessageExecutor);
+
remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManagerActivity, this.heartbeatExecutor);
remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManagerActivity, this.defaultExecutor);
remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManagerActivity, this.defaultExecutor);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
index 904460431..20fab6e57 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
@@ -77,7 +77,7 @@ public class SendMessageActivity extends AbstractRemotingActivity {
}
if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {
if (TopicMessageType.TRANSACTION.equals(messageType)) {
- return sendTransactionMessage(ctx, request, context);
+ messagingProcessor.addTransactionSubscription(context, requestHeader.getProducerGroup(), requestHeader.getTopic());
}
}
return request(ctx, request, context, Duration.ofSeconds(3).toMillis());
@@ -87,10 +87,4 @@ public class SendMessageActivity extends AbstractRemotingActivity {
ProxyContext context) throws Exception {
return request(ctx, request, context, Duration.ofSeconds(3).toMillis());
}
-
- protected RemotingCommand sendTransactionMessage(ChannelHandlerContext ctx, RemotingCommand request,
- ProxyContext context) throws Exception {
- // TODO: wait for connection implement.
- return null;
- }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
new file mode 100644
index 000000000..24f98a875
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.processor.TransactionStatus;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class TransactionActivity extends AbstractRemotingActivity {
+
+ public TransactionActivity(RequestPipeline requestPipeline,
+ MessagingProcessor messagingProcessor) {
+ super(requestPipeline, messagingProcessor);
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+
+ final EndTransactionRequestHeader requestHeader = (EndTransactionRequestHeader) request.decodeCommandCustomHeader(EndTransactionRequestHeader.class);
+
+ TransactionStatus transactionStatus = TransactionStatus.UNKNOWN;
+ switch (requestHeader.getCommitOrRollback()) {
+ case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
+ transactionStatus = TransactionStatus.COMMIT;
+ break;
+ case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
+ transactionStatus = TransactionStatus.ROLLBACK;
+ break;
+ default:
+ break;
+ }
+
+ this.messagingProcessor.endTransaction(
+ context,
+ requestHeader.getTransactionId(),
+ requestHeader.getMsgId(),
+ requestHeader.getProducerGroup(),
+ transactionStatus,
+ requestHeader.getFromTransactionCheck()
+ );
+ return response;
+ }
+}
[rocketmq] 19/26: [ISSUE #5392] Adapt for logging and module refector
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 416427de1ec70e00a8bcd881ebd0571906a10d49
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Wed Nov 23 09:49:32 2022 +0800
[ISSUE #5392] Adapt for logging and module refector
---
.../rocketmq/broker/client/ConsumerManager.java | 2 +-
.../broker/client/ConsumerManagerInterface.java | 6 ++--
.../rocketmq/broker/client/ProducerManager.java | 2 +-
.../broker/client/ProducerManagerInterface.java | 2 +-
.../apache/rocketmq/proxy/config/ProxyConfig.java | 3 +-
.../processor/channel/RemoteChannelSerializer.java | 6 ++--
.../remoting/MultiProtocolRemotingServer.java | 4 +--
.../proxy/remoting/MultiProtocolTlsHelper.java | 4 +--
.../proxy/remoting/RemotingProtocolServer.java | 8 +++---
.../activity/AbstractRemotingActivity.java | 14 +++++-----
.../remoting/activity/ClientManagerActivity.java | 14 +++++-----
.../remoting/activity/ConsumerManagerActivity.java | 14 +++++-----
.../remoting/activity/GetTopicRouteActivity.java | 6 ++--
.../remoting/activity/PopMessageActivity.java | 2 +-
.../remoting/activity/PullMessageActivity.java | 6 ++--
.../remoting/activity/SendMessageActivity.java | 6 ++--
.../remoting/activity/TransactionActivity.java | 4 +--
.../proxy/remoting/channel/RemotingChannel.java | 32 +++++++++++-----------
.../remoting/channel/RemotingChannelManager.java | 6 ++--
.../proxy/remoting/common/RemotingConverter.java | 6 ++--
.../http2proxy/Http2ProtocolProxyHandler.java | 6 ++--
.../http2proxy/Http2ProxyBackendHandler.java | 6 ++--
.../http2proxy/Http2ProxyFrontendHandler.java | 6 ++--
.../rocketmq/proxy/service/admin/AdminService.java | 2 +-
.../proxy/service/admin/DefaultAdminService.java | 10 +++----
.../service/client/ClusterConsumerManager.java | 6 ++--
.../sysmessage/AbstractSystemMessageSyncer.java | 10 +++----
.../proxy/service/sysmessage/HeartbeatSyncer.java | 6 ++--
.../service/sysmessage/HeartbeatSyncerData.java | 6 ++--
.../grpc/v2/channel/GrpcClientChannelTest.java | 4 +--
.../activity/AbstractRemotingActivityTest.java | 8 +++---
.../remoting/activity/PullMessageActivityTest.java | 12 ++++----
.../remoting/activity/SendMessageActivityTest.java | 10 +++----
.../remoting/channel/RemotingChannelTest.java | 14 +++++-----
.../service/admin/DefaultAdminServiceTest.java | 6 ++--
.../service/sysmessage/HeartbeatSyncerTest.java | 18 ++++++------
36 files changed, 139 insertions(+), 138 deletions(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
index 0582ce75e..a70e8579e 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
@@ -38,7 +38,7 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
-public class ConsumerManager {
+public class ConsumerManager implements ConsumerManagerInterface {
private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final ConcurrentMap<String, ConsumerGroupInfo> consumerTable =
new ConcurrentHashMap<>(1024);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java
index 895a2e491..6998f60e7 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java
@@ -20,9 +20,9 @@ package org.apache.rocketmq.broker.client;
import io.netty.channel.Channel;
import java.util.Set;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
-import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
-import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
public interface ConsumerManagerInterface {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
index 047aa8be9..a3ed9c590 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
@@ -35,7 +35,7 @@ import org.apache.rocketmq.remoting.protocol.body.ProducerInfo;
import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
-public class ProducerManager {
+public class ProducerManager implements ProducerManagerInterface {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;
private static final int GET_AVAILABLE_CHANNEL_RETRY_COUNT = 3;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java
index 3f2ece7cd..5e2e7e5b0 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java
@@ -18,7 +18,7 @@
package org.apache.rocketmq.broker.client;
import io.netty.channel.Channel;
-import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
+import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;
public interface ProducerManagerInterface {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
index bd7cf1113..e0f971202 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
@@ -31,6 +31,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.utils.NetworkUtil;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.ProxyMode;
@@ -241,7 +242,7 @@ public class ProxyConfig implements ConfigFile {
public void initData() {
parseDelayLevel();
if (StringUtils.isEmpty(localServeAddr)) {
- this.localServeAddr = RemotingUtil.getLocalAddress();
+ this.localServeAddr = NetworkUtil.getLocalAddress();
}
if (StringUtils.isBlank(localServeAddr)) {
throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "get local serve ip failed");
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java
index 8fd216219..a22401a5f 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelSerializer.java
@@ -23,11 +23,11 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
public class RemoteChannelSerializer {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
private static final String REMOTE_PROXY_IP_KEY = "remoteProxyIp";
private static final String REMOTE_ADDRESS_KEY = "remoteAddress";
private static final String LOCAL_ADDRESS_KEY = "localAddress";
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
index 02e3a545e..74fb3616c 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
@@ -23,6 +23,8 @@ import io.netty.handler.timeout.IdleStateHandler;
import java.io.IOException;
import java.security.cert.CertificateException;
import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.ProxyException;
import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
import org.apache.rocketmq.proxy.remoting.protocol.ProtocolNegotiationHandler;
@@ -33,8 +35,6 @@ import org.apache.rocketmq.remoting.common.TlsMode;
import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* support remoting and http2 protocol at one port
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
index 54af7bc9e..59342ca3c 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
@@ -34,9 +34,9 @@ import java.nio.file.Paths;
import java.security.cert.CertificateException;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.netty.TlsHelper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerAuthClient;
import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerCertPath;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
index adbc21169..a7cc7af47 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -32,10 +32,10 @@ import org.apache.rocketmq.acl.plain.PlainAccessValidator;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.future.FutureTaskExt;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
import org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.config.ProxyConfig;
@@ -59,8 +59,8 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.netty.RequestTask;
import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOutClient {
private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
index 650c38614..a66ee6e04 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
@@ -25,8 +25,9 @@ import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageConst;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.utils.NetworkUtil;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.common.ProxyException;
import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
@@ -35,11 +36,10 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.config.ProxyConfig;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
-import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
public abstract class AbstractRemotingActivity implements NettyRequestProcessor {
protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
@@ -119,8 +119,8 @@ public abstract class AbstractRemotingActivity implements NettyRequestProcessor
context.setAction("Remoting" + request.getCode())
.setLanguage(request.getLanguage().name())
.setChannel(ctx.channel())
- .setLocalAddress(RemotingUtil.socketAddress2String(ctx.channel().localAddress()))
- .setRemoteAddress(RemotingUtil.socketAddress2String(ctx.channel().remoteAddress()));
+ .setLocalAddress(NetworkUtil.socketAddress2String(ctx.channel().localAddress()))
+ .setRemoteAddress(NetworkUtil.socketAddress2String(ctx.channel().remoteAddress()));
return context;
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
index 10f7fa324..1009e4204 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
@@ -25,13 +25,13 @@ import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
import org.apache.rocketmq.broker.client.ProducerChangeListener;
import org.apache.rocketmq.broker.client.ProducerGroupEvent;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader;
-import org.apache.rocketmq.common.protocol.header.UnregisterClientResponseHeader;
-import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
-import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
-import org.apache.rocketmq.common.protocol.heartbeat.ProducerData;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UnregisterClientResponseHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ProducerData;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
index 734b1dad1..1c1993ff0 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
@@ -24,13 +24,13 @@ import java.util.List;
import java.util.Set;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
-import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody;
-import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader;
-import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody;
-import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseHeader;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
index 670d9735c..9972c26c9 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
@@ -23,9 +23,9 @@ import io.netty.channel.ChannelHandlerContext;
import java.util.ArrayList;
import java.util.List;
import org.apache.rocketmq.common.MQVersion;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader;
-import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
import org.apache.rocketmq.proxy.common.Address;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java
index d52b84b12..a635e55cc 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java
@@ -19,7 +19,7 @@ package org.apache.rocketmq.proxy.remoting.activity;
import io.netty.channel.ChannelHandlerContext;
import java.time.Duration;
-import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
index eb744676a..d548ddc0d 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
@@ -20,14 +20,14 @@ package org.apache.rocketmq.proxy.remoting.activity;
import io.netty.channel.ChannelHandlerContext;
import java.time.Duration;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.sysflag.PullSysFlag;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
public class PullMessageActivity extends AbstractRemotingActivity {
public PullMessageActivity(RequestPipeline requestPipeline,
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
index 20fab6e57..618d45874 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
@@ -22,9 +22,9 @@ import java.time.Duration;
import java.util.Map;
import org.apache.rocketmq.common.attribute.TopicMessageType;
import org.apache.rocketmq.common.message.MessageDecoder;
-import org.apache.rocketmq.common.protocol.NamespaceUtil;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
index 24f98a875..bc5e0ca35 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
@@ -18,8 +18,8 @@
package org.apache.rocketmq.proxy.remoting.activity;
import io.netty.channel.ChannelHandlerContext;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
index 806b35de2..368330115 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
@@ -27,16 +27,9 @@ import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult;
-import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
-import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader;
-import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
-import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.common.utils.NetworkUtil;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
import org.apache.rocketmq.proxy.common.utils.FutureUtils;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
@@ -50,12 +43,19 @@ import org.apache.rocketmq.proxy.service.relay.ProxyChannel;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
import org.apache.rocketmq.proxy.service.transaction.TransactionData;
-import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;
+import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
public class RemotingChannel extends ProxyChannel implements RemoteChannelConverter, ChannelExtendAttributeGetter {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
private static final long DEFAULT_MQ_CLIENT_TIMEOUT = Duration.ofSeconds(3).toMillis();
private final String clientId;
private final String remoteAddress;
@@ -67,12 +67,12 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver
Channel parent,
String clientId, Set<SubscriptionData> subscriptionData) {
super(proxyRelayService, parent, parent.id(),
- RemotingUtil.socketAddress2String(parent.remoteAddress()),
- RemotingUtil.socketAddress2String(parent.localAddress()));
+ NetworkUtil.socketAddress2String(parent.remoteAddress()),
+ NetworkUtil.socketAddress2String(parent.localAddress()));
this.remotingProxyOutClient = remotingProxyOutClient;
this.clientId = clientId;
- this.remoteAddress = RemotingUtil.socketAddress2String(parent.remoteAddress());
- this.localAddress = RemotingUtil.socketAddress2String(parent.localAddress());
+ this.remoteAddress = NetworkUtil.socketAddress2String(parent.remoteAddress());
+ this.localAddress = NetworkUtil.socketAddress2String(parent.localAddress());
this.subscriptionData = subscriptionData;
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
index d47884b61..6913fc670 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
@@ -26,12 +26,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
import org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
public class RemotingChannelManager implements StartAndShutdown {
protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java
index a08abbba2..2bd53d8de 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java
@@ -20,11 +20,11 @@ package org.apache.rocketmq.proxy.remoting.common;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
public class RemotingConverter {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
protected static final Object INSTANCE_CREATE_LOCK = new Object();
protected static volatile RemotingConverter instance;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
index 86f1ee921..2ba2d3463 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
@@ -31,15 +31,15 @@ import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import javax.net.ssl.SSLException;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.config.ProxyConfig;
import org.apache.rocketmq.proxy.remoting.protocol.ProtocolHandler;
import org.apache.rocketmq.remoting.common.RemotingHelper;
public class Http2ProtocolProxyHandler implements ProtocolHandler {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
+ private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
private static final String LOCAL_HOST = "127.0.0.1";
/**
* The int value of "PRI ". Now use 4 bytes to judge protocol, may be has potential risks if there is a new protocol
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java
index 53bddfc31..dfcd144af 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java
@@ -22,12 +22,12 @@ import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
public class Http2ProxyBackendHandler extends ChannelInboundHandlerAdapter {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
+ private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
private final Channel inboundChannel;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java
index 8bffdc6d0..775c047b8 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java
@@ -23,12 +23,12 @@ import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
public class Http2ProxyFrontendHandler extends ChannelInboundHandlerAdapter {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
+ private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
// As we use inboundChannel.eventLoop() when building the Bootstrap this does not need to be volatile as
// the outboundChannel will use the same EventLoop (and therefore Thread) as the inboundChannel.
private final Channel outboundChannel;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java
index d98d17ff7..a9e6686b4 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java
@@ -18,7 +18,7 @@
package org.apache.rocketmq.proxy.service.admin;
import java.util.List;
-import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
public interface AdminService {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
index e94c0879b..4dbf21a98 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
@@ -25,17 +25,17 @@ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName;
-import org.apache.rocketmq.common.protocol.route.BrokerData;
-import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIExt;
import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
import org.apache.rocketmq.proxy.service.route.TopicRouteHelper;
public class DefaultAdminService implements AdminService {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
private final MQClientAPIFactory mqClientAPIFactory;
public DefaultAdminService(MQClientAPIFactory mqClientAPIFactory) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
index 3bb65b03e..3a98b5ee1 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
@@ -23,9 +23,9 @@ import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
import org.apache.rocketmq.broker.client.ConsumerManager;
import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
-import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
-import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
import org.apache.rocketmq.proxy.service.admin.AdminService;
import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
index 72593525e..e0a9fd702 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
@@ -29,11 +29,9 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageDecoder;
-import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
-import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.ProxyException;
import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
@@ -44,9 +42,11 @@ import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
import org.apache.rocketmq.proxy.service.route.TopicRouteService;
import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
public abstract class AbstractSystemMessageSyncer implements StartAndShutdown, MessageListenerConcurrently {
- protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ protected static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
protected final TopicRouteService topicRouteService;
protected final AdminService adminService;
protected final String systemResourceName;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
index 12504a2f0..041cbcee6 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
@@ -33,9 +33,9 @@ import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
-import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
index f3b96ac9a..10c6f1206 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
@@ -20,9 +20,9 @@ package org.apache.rocketmq.proxy.service.sysmessage;
import com.google.common.base.MoreObjects;
import java.util.Set;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
-import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
-import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.remoting.protocol.LanguageCode;
public class HeartbeatSyncerData {
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java
index 70e10bc2b..1bdbdd9be 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java
@@ -22,7 +22,7 @@ import apache.rocketmq.v2.Resource;
import apache.rocketmq.v2.Settings;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.rocketmq.proxy.common.ProxyContext;
-import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.config.InitConfigTest;
import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
@@ -41,7 +41,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
-public class GrpcClientChannelTest extends InitConfigAndLoggerTest {
+public class GrpcClientChannelTest extends InitConfigTest {
@Mock
private ProxyRelayService proxyRelayService;
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
index 74eb3cbd8..ecdc1deaf 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
@@ -24,12 +24,12 @@ import java.util.concurrent.CompletableFuture;
import org.apache.rocketmq.acl.common.AclException;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.common.ProxyException;
import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
-import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.config.InitConfigTest;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
@@ -53,7 +53,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
-public class AbstractRemotingActivityTest extends InitConfigAndLoggerTest {
+public class AbstractRemotingActivityTest extends InitConfigTest {
AbstractRemotingActivity remotingActivity;
@Mock
MessagingProcessor messagingProcessorMock;
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
index ffbe2ffac..5798e883b 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
@@ -22,16 +22,16 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPromise;
import java.util.concurrent.CompletableFuture;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.sysflag.PullSysFlag;
-import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.config.InitConfigTest;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -49,7 +49,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
-public class PullMessageActivityTest extends InitConfigAndLoggerTest {
+public class PullMessageActivityTest extends InitConfigTest {
PullMessageActivity pullMessageActivity;
@Mock
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java
index e03bc26e0..b88f6677e 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java
@@ -24,10 +24,10 @@ import java.util.concurrent.CompletableFuture;
import org.apache.rocketmq.common.attribute.TopicMessageType;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageDecoder;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
-import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.proxy.config.InitConfigTest;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
@@ -49,7 +49,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
-public class SendMessageActivityTest extends InitConfigAndLoggerTest {
+public class SendMessageActivityTest extends InitConfigTest {
SendMessageActivity sendMessageActivity;
@Mock
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java
index 840f3e40f..d947fa5d5 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java
@@ -21,15 +21,15 @@ import io.netty.channel.Channel;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.rocketmq.common.filter.FilterAPI;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
-import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.common.utils.NetworkUtil;
+import org.apache.rocketmq.proxy.config.InitConfigTest;
import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;
import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
import org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;
import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
-import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.apache.rocketmq.remoting.protocol.filter.FilterAPI;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -42,7 +42,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
-public class RemotingChannelTest extends InitConfigAndLoggerTest {
+public class RemotingChannelTest extends InitConfigTest {
@Mock
private RemotingProxyOutClient remotingProxyOutClient;
@Mock
@@ -61,8 +61,8 @@ public class RemotingChannelTest extends InitConfigAndLoggerTest {
public void before() throws Throwable {
super.before();
this.clientId = RandomStringUtils.randomAlphabetic(10);
- when(parent.remoteAddress()).thenReturn(RemotingUtil.string2SocketAddress(remoteAddress));
- when(parent.localAddress()).thenReturn(RemotingUtil.string2SocketAddress(localAddress));
+ when(parent.remoteAddress()).thenReturn(NetworkUtil.string2SocketAddress(remoteAddress));
+ when(parent.localAddress()).thenReturn(NetworkUtil.string2SocketAddress(localAddress));
this.subscriptionData = new HashSet<>();
this.subscriptionData.add(FilterAPI.buildSubscriptionData("topic", "subTag"));
this.remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService,
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java
index 039efd8b4..f0e618d11 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java
@@ -22,9 +22,9 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.TopicConfig;
-import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.route.BrokerData;
-import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIExt;
import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
import org.junit.Before;
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
index 8ac74f533..45e3942d6 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
@@ -38,18 +38,11 @@ import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.client.producer.SendStatus;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
-import org.apache.rocketmq.common.filter.FilterAPI;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
-import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
-import org.apache.rocketmq.common.protocol.route.BrokerData;
-import org.apache.rocketmq.common.protocol.route.QueueData;
-import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
-import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.config.InitConfigTest;
import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;
import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;
import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
@@ -63,6 +56,13 @@ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
import org.apache.rocketmq.proxy.service.route.MessageQueueView;
import org.apache.rocketmq.proxy.service.route.TopicRouteService;
import org.apache.rocketmq.remoting.protocol.LanguageCode;
+import org.apache.rocketmq.remoting.protocol.filter.FilterAPI;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.QueueData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
import org.assertj.core.util.Lists;
import org.jetbrains.annotations.NotNull;
import org.junit.Before;
@@ -88,7 +88,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
-public class HeartbeatSyncerTest extends InitConfigAndLoggerTest {
+public class HeartbeatSyncerTest extends InitConfigTest {
@Mock
private TopicRouteService topicRouteService;
@Mock
[rocketmq] 06/26: [ISSUE #5486] Add remoting server
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 48673af7641418d3ae8cd02aa5c341e552036900
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Wed Nov 9 14:53:42 2022 +0800
[ISSUE #5486] Add remoting server
---
.../org/apache/rocketmq/proxy/ProxyStartup.java | 4 +
.../rocketmq/proxy/common/ReflectionCache.java | 45 +++
.../apache/rocketmq/proxy/config/ProxyConfig.java | 196 +++++++++++++
.../remoting/MultiProtocolRemotingServer.java | 133 +++++++++
.../proxy/remoting/MultiProtocolTlsHelper.java | 113 ++++++++
.../proxy/remoting/RemotingProtocolServer.java | 304 ++++++++++++++++++++-
.../proxy/remoting/protocol/ProtocolHandler.java | 28 ++
.../protocol/ProtocolNegotiationHandler.java | 61 +++++
.../http2proxy/Http2ProtocolProxyHandler.java | 119 ++++++++
.../http2proxy/Http2ProxyBackendHandler.java | 67 +++++
.../http2proxy/Http2ProxyFrontendHandler.java | 78 ++++++
.../protocol/remoting/RemotingProtocolHandler.java | 55 ++++
12 files changed, 1199 insertions(+), 4 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
index 78399cf35..42a833430 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
@@ -47,6 +47,7 @@ import org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication;
import org.apache.rocketmq.proxy.metrics.ProxyMetricsManager;
import org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.RemotingProtocolServer;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.srvutil.ServerUtil;
@@ -83,6 +84,9 @@ public class ProxyStartup {
.build();
PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer);
+ RemotingProtocolServer remotingServer = new RemotingProtocolServer(messagingProcessor);
+ PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(remotingServer);
+
// start servers one by one.
PROXY_START_AND_SHUTDOWN.start();
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReflectionCache.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReflectionCache.java
new file mode 100644
index 000000000..31fa46c90
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReflectionCache.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import java.lang.reflect.Field;
+import java.util.concurrent.TimeUnit;
+
+public class ReflectionCache {
+ private final Cache<Class<?>, Field> fieldCache;
+ private static final int DEFAULT_MAX_SIZE = 15;
+
+ public ReflectionCache() {
+ this(DEFAULT_MAX_SIZE);
+ }
+
+ public ReflectionCache(int maxSize) {
+ this.fieldCache = CacheBuilder.newBuilder().maximumSize(maxSize).expireAfterAccess(5, TimeUnit.MINUTES).build();
+ }
+
+ public Field getDeclaredField(final Class<?> clazz, final String fieldName) throws Exception {
+ return this.fieldCache.get(clazz, () -> {
+ Field field = clazz.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ return field;
+ });
+ }
+}
+
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
index 0efca05b4..b613c191e 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
@@ -209,6 +209,33 @@ public class ProxyConfig implements ConfigFile {
private long channelExpiredTimeout = 1000 * 120;
+ // remoting
+
+ private boolean enableRemotingLocalProxyGrpc = true;
+ private int localProxyConnectTimeoutMs = 3000;
+ private int remotingListenPort = 8080;
+
+ private int remotingHeartbeatThreadPoolNums = 2 * PROCESSOR_NUMBER;
+ private int remotingTopicRouteThreadPoolNums = 2 * PROCESSOR_NUMBER;
+ private int remotingSendMessageThreadPoolNums = 4 * PROCESSOR_NUMBER;
+ private int remotingPullMessageThreadPoolNums = 4 * PROCESSOR_NUMBER;
+ private int remotingUpdateOffsetThreadPoolNums = 4 * PROCESSOR_NUMBER;
+ private int remotingDefaultThreadPoolNums = 4 * PROCESSOR_NUMBER;
+
+ private int remotingHeartbeatThreadPoolQueueCapacity = 50000;
+ private int remotingTopicRouteThreadPoolQueueCapacity = 50000;
+ private int remotingSendThreadPoolQueueCapacity = 10000;
+ private int remotingPullThreadPoolQueueCapacity = 50000;
+ private int remotingUpdateOffsetThreadPoolQueueCapacity = 10000;
+ private int remotingDefaultThreadPoolQueueCapacity = 50000;
+
+ private long remotingWaitTimeMillsInSendQueue = 3 * 1000;
+ private long remotingWaitTimeMillsInPullQueue = 5 * 1000;
+ private long remotingWaitTimeMillsInHeartbeatQueue = 31 * 1000;
+ private long remotingWaitTimeMillsInUpdateOffsetQueue = 3 * 1000;
+ private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000;
+ private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000;
+
@Override
public void initData() {
parseDelayLevel();
@@ -1124,7 +1151,176 @@ public class ProxyConfig implements ConfigFile {
return channelExpiredTimeout;
}
+ public boolean isEnableRemotingLocalProxyGrpc() {
+ return enableRemotingLocalProxyGrpc;
+ }
+
public void setChannelExpiredTimeout(long channelExpiredTimeout) {
this.channelExpiredTimeout = channelExpiredTimeout;
}
}
+
+ public void setEnableRemotingLocalProxyGrpc(boolean enableRemotingLocalProxyGrpc) {
+ this.enableRemotingLocalProxyGrpc = enableRemotingLocalProxyGrpc;
+ }
+
+ public int getLocalProxyConnectTimeoutMs() {
+ return localProxyConnectTimeoutMs;
+ }
+
+ public void setLocalProxyConnectTimeoutMs(int localProxyConnectTimeoutMs) {
+ this.localProxyConnectTimeoutMs = localProxyConnectTimeoutMs;
+ }
+
+ public int getRemotingListenPort() {
+ return remotingListenPort;
+ }
+
+ public void setRemotingListenPort(int remotingListenPort) {
+ this.remotingListenPort = remotingListenPort;
+ }
+
+ public int getRemotingHeartbeatThreadPoolNums() {
+ return remotingHeartbeatThreadPoolNums;
+ }
+
+ public void setRemotingHeartbeatThreadPoolNums(int remotingHeartbeatThreadPoolNums) {
+ this.remotingHeartbeatThreadPoolNums = remotingHeartbeatThreadPoolNums;
+ }
+
+ public int getRemotingTopicRouteThreadPoolNums() {
+ return remotingTopicRouteThreadPoolNums;
+ }
+
+ public void setRemotingTopicRouteThreadPoolNums(int remotingTopicRouteThreadPoolNums) {
+ this.remotingTopicRouteThreadPoolNums = remotingTopicRouteThreadPoolNums;
+ }
+
+ public int getRemotingSendMessageThreadPoolNums() {
+ return remotingSendMessageThreadPoolNums;
+ }
+
+ public void setRemotingSendMessageThreadPoolNums(int remotingSendMessageThreadPoolNums) {
+ this.remotingSendMessageThreadPoolNums = remotingSendMessageThreadPoolNums;
+ }
+
+ public int getRemotingPullMessageThreadPoolNums() {
+ return remotingPullMessageThreadPoolNums;
+ }
+
+ public void setRemotingPullMessageThreadPoolNums(int remotingPullMessageThreadPoolNums) {
+ this.remotingPullMessageThreadPoolNums = remotingPullMessageThreadPoolNums;
+ }
+
+ public int getRemotingUpdateOffsetThreadPoolNums() {
+ return remotingUpdateOffsetThreadPoolNums;
+ }
+
+ public void setRemotingUpdateOffsetThreadPoolNums(int remotingUpdateOffsetThreadPoolNums) {
+ this.remotingUpdateOffsetThreadPoolNums = remotingUpdateOffsetThreadPoolNums;
+ }
+
+ public int getRemotingDefaultThreadPoolNums() {
+ return remotingDefaultThreadPoolNums;
+ }
+
+ public void setRemotingDefaultThreadPoolNums(int remotingDefaultThreadPoolNums) {
+ this.remotingDefaultThreadPoolNums = remotingDefaultThreadPoolNums;
+ }
+
+ public int getRemotingHeartbeatThreadPoolQueueCapacity() {
+ return remotingHeartbeatThreadPoolQueueCapacity;
+ }
+
+ public void setRemotingHeartbeatThreadPoolQueueCapacity(int remotingHeartbeatThreadPoolQueueCapacity) {
+ this.remotingHeartbeatThreadPoolQueueCapacity = remotingHeartbeatThreadPoolQueueCapacity;
+ }
+
+ public int getRemotingTopicRouteThreadPoolQueueCapacity() {
+ return remotingTopicRouteThreadPoolQueueCapacity;
+ }
+
+ public void setRemotingTopicRouteThreadPoolQueueCapacity(int remotingTopicRouteThreadPoolQueueCapacity) {
+ this.remotingTopicRouteThreadPoolQueueCapacity = remotingTopicRouteThreadPoolQueueCapacity;
+ }
+
+ public int getRemotingSendThreadPoolQueueCapacity() {
+ return remotingSendThreadPoolQueueCapacity;
+ }
+
+ public void setRemotingSendThreadPoolQueueCapacity(int remotingSendThreadPoolQueueCapacity) {
+ this.remotingSendThreadPoolQueueCapacity = remotingSendThreadPoolQueueCapacity;
+ }
+
+ public int getRemotingPullThreadPoolQueueCapacity() {
+ return remotingPullThreadPoolQueueCapacity;
+ }
+
+ public void setRemotingPullThreadPoolQueueCapacity(int remotingPullThreadPoolQueueCapacity) {
+ this.remotingPullThreadPoolQueueCapacity = remotingPullThreadPoolQueueCapacity;
+ }
+
+ public int getRemotingUpdateOffsetThreadPoolQueueCapacity() {
+ return remotingUpdateOffsetThreadPoolQueueCapacity;
+ }
+
+ public void setRemotingUpdateOffsetThreadPoolQueueCapacity(int remotingUpdateOffsetThreadPoolQueueCapacity) {
+ this.remotingUpdateOffsetThreadPoolQueueCapacity = remotingUpdateOffsetThreadPoolQueueCapacity;
+ }
+
+ public int getRemotingDefaultThreadPoolQueueCapacity() {
+ return remotingDefaultThreadPoolQueueCapacity;
+ }
+
+ public void setRemotingDefaultThreadPoolQueueCapacity(int remotingDefaultThreadPoolQueueCapacity) {
+ this.remotingDefaultThreadPoolQueueCapacity = remotingDefaultThreadPoolQueueCapacity;
+ }
+
+ public long getRemotingWaitTimeMillsInSendQueue() {
+ return remotingWaitTimeMillsInSendQueue;
+ }
+
+ public void setRemotingWaitTimeMillsInSendQueue(long remotingWaitTimeMillsInSendQueue) {
+ this.remotingWaitTimeMillsInSendQueue = remotingWaitTimeMillsInSendQueue;
+ }
+
+ public long getRemotingWaitTimeMillsInPullQueue() {
+ return remotingWaitTimeMillsInPullQueue;
+ }
+
+ public void setRemotingWaitTimeMillsInPullQueue(long remotingWaitTimeMillsInPullQueue) {
+ this.remotingWaitTimeMillsInPullQueue = remotingWaitTimeMillsInPullQueue;
+ }
+
+ public long getRemotingWaitTimeMillsInHeartbeatQueue() {
+ return remotingWaitTimeMillsInHeartbeatQueue;
+ }
+
+ public void setRemotingWaitTimeMillsInHeartbeatQueue(long remotingWaitTimeMillsInHeartbeatQueue) {
+ this.remotingWaitTimeMillsInHeartbeatQueue = remotingWaitTimeMillsInHeartbeatQueue;
+ }
+
+ public long getRemotingWaitTimeMillsInUpdateOffsetQueue() {
+ return remotingWaitTimeMillsInUpdateOffsetQueue;
+ }
+
+ public void setRemotingWaitTimeMillsInUpdateOffsetQueue(long remotingWaitTimeMillsInUpdateOffsetQueue) {
+ this.remotingWaitTimeMillsInUpdateOffsetQueue = remotingWaitTimeMillsInUpdateOffsetQueue;
+ }
+
+ public long getRemotingWaitTimeMillsInTopicRouteQueue() {
+ return remotingWaitTimeMillsInTopicRouteQueue;
+ }
+
+ public void setRemotingWaitTimeMillsInTopicRouteQueue(long remotingWaitTimeMillsInTopicRouteQueue) {
+ this.remotingWaitTimeMillsInTopicRouteQueue = remotingWaitTimeMillsInTopicRouteQueue;
+ }
+
+ public long getRemotingWaitTimeMillsInDefaultQueue() {
+ return remotingWaitTimeMillsInDefaultQueue;
+ }
+
+ public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) {
+ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
new file mode 100644
index 000000000..73aeeaf42
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting;
+
+import com.google.common.base.Preconditions;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.timeout.IdleStateHandler;
+import io.netty.util.concurrent.DefaultEventExecutorGroup;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.security.cert.CertificateException;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.proxy.common.ProxyException;
+import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
+import org.apache.rocketmq.proxy.remoting.protocol.ProtocolNegotiationHandler;
+import org.apache.rocketmq.proxy.remoting.protocol.http2proxy.Http2ProtocolProxyHandler;
+import org.apache.rocketmq.proxy.remoting.protocol.remoting.RemotingProtocolHandler;
+import org.apache.rocketmq.remoting.ChannelEventListener;
+import org.apache.rocketmq.remoting.common.TlsMode;
+import org.apache.rocketmq.remoting.netty.NettyEncoder;
+import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * for remoting server, if config listen port is 8080 in nettyServerConfig
+ * <p>
+ * will
+ * <li>listen port at 9080 with protocol remoting</li>
+ * <li>listen port at 8080 with protocol remoting and http2</li>
+ */
+public class MultiProtocolRemotingServer extends NettyRemotingServer {
+
+ private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private static final int PORT_DELTA = 1000;
+ private final NettyServerConfig nettyServerConfig;
+ private final int port;
+
+ public MultiProtocolRemotingServer(NettyServerConfig nettyServerConfig, ChannelEventListener channelEventListener) {
+ super(nettyServerConfig, channelEventListener);
+ this.port = nettyServerConfig.getListenPort();
+ // to support multiple protocol
+ // will bind the real port in configChildHandler
+ // so let parent bind to a useless port
+ nettyServerConfig.setListenPort(nettyServerConfig.getListenPort() + PORT_DELTA);
+ this.nettyServerConfig = nettyServerConfig;
+ }
+
+ @Override
+ public void loadSslContext() {
+ TlsMode tlsMode = TlsSystemConfig.tlsMode;
+ log.info("Server is running in TLS {} mode", tlsMode.getName());
+
+ if (tlsMode != TlsMode.DISABLED) {
+ try {
+ sslContext = MultiProtocolTlsHelper.buildSslContext();
+ log.info("SSLContext created for server");
+ } catch (CertificateException | IOException e) {
+ throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "Failed to create SSLContext for server", e);
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ super.start();
+ this.configChildHandler();
+ }
+
+ protected void configChildHandler() {
+ try {
+ ServerBootstrap serverBootstrap = getField("serverBootstrap", ServerBootstrap.class);
+ Preconditions.checkNotNull(serverBootstrap);
+ DefaultEventExecutorGroup defaultEventExecutorGroup = getField("defaultEventExecutorGroup", DefaultEventExecutorGroup.class);
+ Preconditions.checkNotNull(defaultEventExecutorGroup);
+ NettyEncoder encoder = getField("encoder", NettyEncoder.class);
+ Preconditions.checkNotNull(encoder);
+ ChannelDuplexHandler connectionManageHandler = getField("connectionManageHandler", ChannelDuplexHandler.class);
+ Preconditions.checkNotNull(connectionManageHandler);
+ SimpleChannelInboundHandler serverHandler = getField("serverHandler", SimpleChannelInboundHandler.class);
+ Preconditions.checkNotNull(serverHandler);
+ SimpleChannelInboundHandler handshakeHandler = getField("handshakeHandler", SimpleChannelInboundHandler.class);
+ Preconditions.checkNotNull(handshakeHandler);
+ ConcurrentMap remotingServerTable = getField("remotingServerTable", ConcurrentMap.class);
+ Preconditions.checkNotNull(remotingServerTable);
+
+ serverBootstrap.childHandler(new ChannelInitializer<SocketChannel>() {
+ @Override
+ protected void initChannel(SocketChannel ch) {
+ ch.pipeline()
+ .addLast(defaultEventExecutorGroup, "handshakeHandler", handshakeHandler)
+ .addLast(defaultEventExecutorGroup,
+ new IdleStateHandler(0, 0, nettyServerConfig.getServerChannelMaxIdleTimeSeconds()),
+ new ProtocolNegotiationHandler(new RemotingProtocolHandler(encoder, connectionManageHandler, serverHandler))
+ .addProtocolHandler(new Http2ProtocolProxyHandler())
+ );
+ }
+ });
+ remotingServerTable.put(port, this);
+ serverBootstrap.bind(port).sync();
+ } catch (Throwable t) {
+ throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "config netty child handler failed", t);
+ }
+ }
+
+ protected <T> T getField(String name, Class<T> getClazz) throws Throwable {
+ Field field = NettyRemotingServer.class.getDeclaredField(name);
+ field.setAccessible(true);
+ return getClazz.cast(field.get(this));
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
new file mode 100644
index 000000000..54af7bc9e
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting;
+
+import io.netty.handler.ssl.ApplicationProtocolConfig;
+import io.netty.handler.ssl.ApplicationProtocolNames;
+import io.netty.handler.ssl.ClientAuth;
+import io.netty.handler.ssl.OpenSsl;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.cert.CertificateException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.remoting.netty.TlsHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerAuthClient;
+import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerCertPath;
+import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerKeyPassword;
+import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerKeyPath;
+import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerNeedClientAuth;
+import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsServerTrustCertPath;
+import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.tlsTestModeEnable;
+
+public class MultiProtocolTlsHelper extends TlsHelper {
+ private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ private static final DecryptionStrategy DECRYPTION_STRATEGY = (privateKeyEncryptPath, forClient) -> new FileInputStream(privateKeyEncryptPath);
+
+ public static SslContext buildSslContext() throws IOException, CertificateException {
+ TlsHelper.buildSslContext(false);
+ SslProvider provider;
+ if (OpenSsl.isAvailable()) {
+ provider = SslProvider.OPENSSL;
+ log.info("Using OpenSSL provider");
+ } else {
+ provider = SslProvider.JDK;
+ log.info("Using JDK SSL provider");
+ }
+
+ SslContextBuilder sslContextBuilder = null;
+ if (tlsTestModeEnable) {
+ SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate();
+ sslContextBuilder = SslContextBuilder
+ .forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey())
+ .sslProvider(SslProvider.OPENSSL)
+ .clientAuth(ClientAuth.OPTIONAL);
+ } else {
+ sslContextBuilder = SslContextBuilder.forServer(
+ !StringUtils.isBlank(tlsServerCertPath) ? Files.newInputStream(Paths.get(tlsServerCertPath)) : null,
+ !StringUtils.isBlank(tlsServerKeyPath) ? DECRYPTION_STRATEGY.decryptPrivateKey(tlsServerKeyPath, false) : null,
+ !StringUtils.isBlank(tlsServerKeyPassword) ? tlsServerKeyPassword : null)
+ .sslProvider(provider);
+
+ if (!tlsServerAuthClient) {
+ sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE);
+ } else {
+ if (!StringUtils.isBlank(tlsServerTrustCertPath)) {
+ sslContextBuilder.trustManager(new File(tlsServerTrustCertPath));
+ }
+ }
+
+ sslContextBuilder.clientAuth(parseClientAuthMode(tlsServerNeedClientAuth));
+ }
+
+ sslContextBuilder.applicationProtocolConfig(new ApplicationProtocolConfig(
+ ApplicationProtocolConfig.Protocol.ALPN,
+ // NO_ADVERTISE is currently the only mode supported by both OpenSsl and JDK providers.
+ ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,
+ // ACCEPT is currently the only mode supported by both OpenSsl and JDK providers.
+ ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,
+ ApplicationProtocolNames.HTTP_2));
+
+ return sslContextBuilder.build();
+ }
+
+ private static ClientAuth parseClientAuthMode(String authMode) {
+ if (null == authMode || authMode.trim().isEmpty()) {
+ return ClientAuth.NONE;
+ }
+
+ for (ClientAuth clientAuth : ClientAuth.values()) {
+ if (clientAuth.name().equals(authMode.toUpperCase())) {
+ return clientAuth;
+ }
+ }
+
+ return ClientAuth.NONE;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
index 58b257641..fdf1870a5 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -18,20 +18,161 @@
package org.apache.rocketmq.proxy.remoting;
import io.netty.channel.Channel;
+import java.lang.reflect.Field;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.broker.latency.FutureTaskExt;
import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
+import org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor;
+import org.apache.rocketmq.proxy.common.ReflectionCache;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.activity.AckMessageActivity;
+import org.apache.rocketmq.proxy.remoting.activity.ChangeInvisibleTimeActivity;
+import org.apache.rocketmq.proxy.remoting.activity.ClientManagerActivity;
+import org.apache.rocketmq.proxy.remoting.activity.ConsumerManagerActivity;
+import org.apache.rocketmq.proxy.remoting.activity.GetTopicRouteActivity;
+import org.apache.rocketmq.proxy.remoting.activity.PopMessageActivity;
+import org.apache.rocketmq.proxy.remoting.activity.PullMessageActivity;
+import org.apache.rocketmq.proxy.remoting.activity.SendMessageActivity;
+import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.ChannelEventListener;
import org.apache.rocketmq.remoting.RemotingServer;
+import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.netty.RequestTask;
+import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOutClient {
+ private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
- private final MessagingProcessor messagingProcessor;
- private RemotingServer defaultRemotingServer;
+ protected final MessagingProcessor messagingProcessor;
+ protected final RemotingChannelManager remotingChannelManager;
+ protected final ChannelEventListener clientHousekeepingService;
+ protected final RemotingServer defaultRemotingServer;
+ protected final GetTopicRouteActivity getTopicRouteActivity;
+ protected final ClientManagerActivity clientManagerActivity;
+ protected final ConsumerManagerActivity consumerManagerActivity;
+ protected final SendMessageActivity sendMessageActivity;
+ protected final PullMessageActivity pullMessageActivity;
+ protected final PopMessageActivity popMessageActivity;
+ protected final AckMessageActivity ackMessageActivity;
+ protected final ChangeInvisibleTimeActivity changeInvisibleTimeActivity;
+ protected final ThreadPoolExecutor sendMessageExecutor;
+ protected final ThreadPoolExecutor pullMessageExecutor;
+ protected final ThreadPoolExecutor heartbeatExecutor;
+ protected final ThreadPoolExecutor updateOffsetExecutor;
+ protected final ThreadPoolExecutor topicRouteExecutor;
+ protected final ThreadPoolExecutor defaultExecutor;
+
+ private final ReflectionCache reflectionCache = new ReflectionCache();
public RemotingProtocolServer(MessagingProcessor messagingProcessor) {
this.messagingProcessor = messagingProcessor;
+ this.remotingChannelManager = new RemotingChannelManager(this, messagingProcessor.getProxyRelayService());
+
+ RequestPipeline pipeline = createRequestPipeline();
+ this.getTopicRouteActivity = new GetTopicRouteActivity(pipeline, messagingProcessor);
+ this.clientManagerActivity = new ClientManagerActivity(pipeline, messagingProcessor, remotingChannelManager);
+ this.consumerManagerActivity = new ConsumerManagerActivity(pipeline, messagingProcessor);
+ this.sendMessageActivity = new SendMessageActivity(pipeline, messagingProcessor);
+ this.pullMessageActivity = new PullMessageActivity(pipeline, messagingProcessor);
+ this.popMessageActivity = new PopMessageActivity(pipeline, messagingProcessor);
+ this.ackMessageActivity = new AckMessageActivity(pipeline, messagingProcessor);
+ this.changeInvisibleTimeActivity = new ChangeInvisibleTimeActivity(pipeline, messagingProcessor);
+
+ ProxyConfig config = ConfigurationManager.getProxyConfig();
+ NettyServerConfig defaultServerConfig = new NettyServerConfig();
+ defaultServerConfig.setListenPort(config.getRemotingListenPort());
+ TlsSystemConfig.tlsTestModeEnable = false;
+ System.setProperty(TlsSystemConfig.TLS_TEST_MODE_ENABLE, "false");
+ TlsSystemConfig.tlsServerCertPath = config.getGrpcTlsCertPath();
+ System.setProperty(TlsSystemConfig.TLS_SERVER_CERTPATH, config.getGrpcTlsCertPath());
+ TlsSystemConfig.tlsServerKeyPath = config.getGrpcTlsKeyPath();
+ System.setProperty(TlsSystemConfig.TLS_SERVER_KEYPATH, config.getGrpcTlsKeyPath());
+
+ this.clientHousekeepingService = new ClientHousekeepingService(this.clientManagerActivity);
+
+ if (config.isEnableRemotingLocalProxyGrpc()) {
+ this.defaultRemotingServer = new MultiProtocolRemotingServer(defaultServerConfig, this.clientHousekeepingService);
+ } else {
+ this.defaultRemotingServer = new NettyRemotingServer(defaultServerConfig, this.clientHousekeepingService);
+ }
+ this.registerRemotingServer(this.defaultRemotingServer);
+
+ this.sendMessageExecutor = ThreadPoolMonitor.createAndMonitor(
+ config.getRemotingSendMessageThreadPoolNums(),
+ config.getRemotingSendMessageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ "RemotingSendMessageThread",
+ config.getRemotingSendThreadPoolQueueCapacity(),
+ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInSendQueue())
+ );
+
+ this.pullMessageExecutor = ThreadPoolMonitor.createAndMonitor(
+ config.getRemotingPullMessageThreadPoolNums(),
+ config.getRemotingPullMessageThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ "RemotingPullMessageThread",
+ config.getRemotingPullThreadPoolQueueCapacity(),
+ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInPullQueue())
+ );
+
+ this.updateOffsetExecutor = ThreadPoolMonitor.createAndMonitor(
+ config.getRemotingUpdateOffsetThreadPoolNums(),
+ config.getRemotingUpdateOffsetThreadPoolNums(),
+ 1,
+ TimeUnit.MINUTES,
+ "RemotingUpdateOffsetThread",
+ config.getRemotingUpdateOffsetThreadPoolQueueCapacity(),
+ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInUpdateOffsetQueue())
+ );
+
+ this.heartbeatExecutor = ThreadPoolMonitor.createAndMonitor(
+ config.getRemotingHeartbeatThreadPoolNums(),
+ config.getRemotingHeartbeatThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ "RemotingHeartbeatThread",
+ config.getRemotingHeartbeatThreadPoolQueueCapacity(),
+ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInHeartbeatQueue())
+ );
+
+ this.topicRouteExecutor = ThreadPoolMonitor.createAndMonitor(
+ config.getRemotingTopicRouteThreadPoolNums(),
+ config.getRemotingTopicRouteThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ "RemotingTopicRouteThread",
+ config.getRemotingTopicRouteThreadPoolQueueCapacity(),
+ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInTopicRouteQueue())
+ );
+
+ this.defaultExecutor = ThreadPoolMonitor.createAndMonitor(
+ config.getRemotingDefaultThreadPoolNums(),
+ config.getRemotingDefaultThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MILLISECONDS,
+ "RemotingDefaultThread",
+ config.getRemotingDefaultThreadPoolQueueCapacity(),
+ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInDefaultQueue())
+ );
}
protected void init() {
@@ -39,17 +180,50 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
}
protected void registerRemotingServer(RemotingServer remotingServer) {
+ remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageActivity, this.sendMessageExecutor);
+ remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageActivity, this.sendMessageExecutor);
+ remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageActivity, this.sendMessageExecutor);
+ remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageActivity, sendMessageExecutor);
+
+ remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManagerActivity, this.heartbeatExecutor);
+ remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManagerActivity, this.defaultExecutor);
+ remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManagerActivity, this.defaultExecutor);
+
+ remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, pullMessageActivity, this.pullMessageExecutor);
+ remotingServer.registerProcessor(RequestCode.LITE_PULL_MESSAGE, pullMessageActivity, this.pullMessageExecutor);
+ remotingServer.registerProcessor(RequestCode.POP_MESSAGE, pullMessageActivity, this.pullMessageExecutor);
+ remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManagerActivity, this.updateOffsetExecutor);
+ remotingServer.registerProcessor(RequestCode.ACK_MESSAGE, consumerManagerActivity, this.updateOffsetExecutor);
+ remotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, consumerManagerActivity, this.updateOffsetExecutor);
+
+ remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManagerActivity, this.defaultExecutor);
+ remotingServer.registerProcessor(RequestCode.GET_MAX_OFFSET, consumerManagerActivity, this.defaultExecutor);
+ remotingServer.registerProcessor(RequestCode.GET_MIN_OFFSET, consumerManagerActivity, this.defaultExecutor);
+ remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManagerActivity, this.defaultExecutor);
+ remotingServer.registerProcessor(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, consumerManagerActivity, this.defaultExecutor);
+ remotingServer.registerProcessor(RequestCode.LOCK_BATCH_MQ, consumerManagerActivity, this.defaultExecutor);
+ remotingServer.registerProcessor(RequestCode.UNLOCK_BATCH_MQ, consumerManagerActivity, this.defaultExecutor);
+
+ remotingServer.registerProcessor(RequestCode.GET_ROUTEINFO_BY_TOPIC, getTopicRouteActivity, this.topicRouteExecutor);
}
@Override
public void shutdown() throws Exception {
-
+ this.defaultRemotingServer.shutdown();
+ this.remotingChannelManager.shutdown();
+ this.sendMessageExecutor.shutdown();
+ this.pullMessageExecutor.shutdown();
+ this.heartbeatExecutor.shutdown();
+ this.updateOffsetExecutor.shutdown();
+ this.topicRouteExecutor.shutdown();
+ this.defaultExecutor.shutdown();
}
@Override
public void start() throws Exception {
-
+ this.remotingChannelManager.start();
+ this.defaultRemotingServer.start();
}
@Override
@@ -69,4 +243,126 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
}
return future;
}
+
+ protected RequestPipeline createRequestPipeline() {
+ RequestPipeline pipeline = (ctx, request, context) -> {
+ };
+
+ // add pipeline
+ // the last pipe add will execute at the first
+ return pipeline;
+ }
+
+ protected class ThreadPoolHeadSlowTimeMillsMonitor implements ThreadPoolStatusMonitor {
+
+ private final long maxWaitTimeMillsInQueue;
+
+ public ThreadPoolHeadSlowTimeMillsMonitor(long maxWaitTimeMillsInQueue) {
+ this.maxWaitTimeMillsInQueue = maxWaitTimeMillsInQueue;
+ }
+
+ @Override
+ public String describe() {
+ return "headSlow";
+ }
+
+ @Override
+ public double value(ThreadPoolExecutor executor) {
+ return headSlowTimeMills(executor.getQueue());
+ }
+
+ @Override
+ public boolean needPrintJstack(ThreadPoolExecutor executor, double value) {
+ return value > maxWaitTimeMillsInQueue;
+ }
+ }
+
+ protected long headSlowTimeMills(BlockingQueue<Runnable> q) {
+ try {
+ long slowTimeMills = 0;
+ final Runnable peek = q.peek();
+ if (peek != null) {
+ RequestTask rt = castRunnable(peek);
+ slowTimeMills = rt == null ? 0 : System.currentTimeMillis() - rt.getCreateTimestamp();
+ }
+
+ if (slowTimeMills < 0) {
+ slowTimeMills = 0;
+ }
+
+ return slowTimeMills;
+ } catch (Exception e) {
+ log.error("error when headSlowTimeMills.", e);
+ }
+ return -1;
+ }
+
+ protected void cleanExpireRequest() {
+ ProxyConfig config = ConfigurationManager.getProxyConfig();
+
+ cleanExpiredRequestInQueue(this.sendMessageExecutor, config.getRemotingWaitTimeMillsInSendQueue());
+ cleanExpiredRequestInQueue(this.pullMessageExecutor, config.getRemotingWaitTimeMillsInPullQueue());
+ cleanExpiredRequestInQueue(this.heartbeatExecutor, config.getRemotingWaitTimeMillsInHeartbeatQueue());
+ cleanExpiredRequestInQueue(this.updateOffsetExecutor, config.getRemotingWaitTimeMillsInUpdateOffsetQueue());
+ cleanExpiredRequestInQueue(this.topicRouteExecutor, config.getRemotingWaitTimeMillsInTopicRouteQueue());
+ cleanExpiredRequestInQueue(this.defaultExecutor, config.getRemotingWaitTimeMillsInDefaultQueue());
+ }
+
+ protected void cleanExpiredRequestInQueue(ThreadPoolExecutor threadPoolExecutor, long maxWaitTimeMillsInQueue) {
+ while (true) {
+ try {
+ BlockingQueue<Runnable> blockingQueue = threadPoolExecutor.getQueue();
+ if (!blockingQueue.isEmpty()) {
+ final Runnable runnable = blockingQueue.peek();
+ if (null == runnable) {
+ break;
+ }
+ final RequestTask rt = castRunnable(runnable);
+ if (rt == null || rt.isStopRun()) {
+ break;
+ }
+
+ final long behind = System.currentTimeMillis() - rt.getCreateTimestamp();
+ if (behind >= maxWaitTimeMillsInQueue) {
+ if (blockingQueue.remove(runnable)) {
+ rt.setStopRun(true);
+ rt.returnResponse(ResponseCode.SYSTEM_BUSY,
+ String.format("[TIMEOUT_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d", behind, blockingQueue.size()));
+ }
+ } else {
+ break;
+ }
+ } else {
+ break;
+ }
+ } catch (Throwable ignored) {
+ }
+ }
+ }
+
+ private RequestTask castRunnable(final Runnable runnable) {
+ try {
+ if (runnable instanceof FutureTask) {
+ Field callableField = reflectionCache.getDeclaredField(FutureTask.class, "callable");
+ Callable callable = (Callable) callableField.get(runnable);
+ if (callable == null) {
+ return null;
+ }
+ Field taskField = reflectionCache.getDeclaredField(callable.getClass(), "task");
+ if (taskField == null) {
+ log.warn("get task from FutureTask failed. class:{}", runnable.getClass().getName());
+ return null;
+ }
+ return (RequestTask) taskField.get(callable);
+ } else if (runnable instanceof FutureTaskExt) {
+ FutureTaskExt futureTaskExt = (FutureTaskExt) runnable;
+ return (RequestTask) futureTaskExt.getRunnable();
+ }
+ return null;
+ } catch (Throwable e) {
+ log.error("castRunnable exception. class:{}", runnable.getClass().getName(), e);
+ }
+
+ return null;
+ }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolHandler.java
new file mode 100644
index 000000000..4b1b03067
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolHandler.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.protocol;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+
+public interface ProtocolHandler {
+
+ boolean match(ByteBuf msg);
+
+ void config(final ChannelHandlerContext ctx, final ByteBuf msg);
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolNegotiationHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolNegotiationHandler.java
new file mode 100644
index 000000000..da2dded5f
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/ProtocolNegotiationHandler.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.remoting.protocol;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.ByteToMessageDecoder;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ProtocolNegotiationHandler extends ByteToMessageDecoder {
+
+ private final List<ProtocolHandler> protocolHandlerList = new ArrayList<ProtocolHandler>();
+ private final ProtocolHandler fallbackProtocolHandler;
+
+ public ProtocolNegotiationHandler(ProtocolHandler fallbackProtocolHandler) {
+ this.fallbackProtocolHandler = fallbackProtocolHandler;
+ }
+
+ public ProtocolNegotiationHandler addProtocolHandler(ProtocolHandler protocolHandler) {
+ protocolHandlerList.add(protocolHandler);
+ return this;
+ }
+
+ @Override
+ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
+ // use 4 bytes to judge protocol
+ if (in.readableBytes() < 4) {
+ return;
+ }
+
+ ProtocolHandler protocolHandler = null;
+ for (ProtocolHandler curProtocolHandler : protocolHandlerList) {
+ if (curProtocolHandler.match(in)) {
+ protocolHandler = curProtocolHandler;
+ break;
+ }
+ }
+
+ if (protocolHandler == null) {
+ protocolHandler = fallbackProtocolHandler;
+ }
+
+ protocolHandler.config(ctx, in);
+ ctx.pipeline().remove(this);
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
new file mode 100644
index 000000000..c5050cda7
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.protocol.http2proxy;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.handler.ssl.ApplicationProtocolConfig;
+import io.netty.handler.ssl.ApplicationProtocolNames;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import javax.net.ssl.SSLException;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.remoting.protocol.ProtocolHandler;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+
+public class Http2ProtocolProxyHandler implements ProtocolHandler {
+ private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
+ private static final String LOCAL_HOST = "127.0.0.1";
+ /**
+ * The int value of "PRI ". Now use 4 bytes to judge protocol, may be has potential risks if there is a new protocol
+ * which start with "PRI " too in the future
+ * <p>
+ * The full HTTP/2 connection preface is "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
+ * <p>
+ * ref: https://datatracker.ietf.org/doc/html/rfc7540#section-3.5
+ */
+ private static final int PRI_INT = 0x50524920;
+
+ private final SslContext sslContext;
+
+ public Http2ProtocolProxyHandler() {
+ try {
+ sslContext = SslContextBuilder
+ .forClient()
+ .sslProvider(SslProvider.OPENSSL)
+ .trustManager(InsecureTrustManagerFactory.INSTANCE)
+ .applicationProtocolConfig(new ApplicationProtocolConfig(
+ ApplicationProtocolConfig.Protocol.ALPN,
+ ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE,
+ ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT,
+ ApplicationProtocolNames.HTTP_2))
+ .build();
+ } catch (SSLException e) {
+ log.error("Failed to create SSLContext for Http2ProtocolProxyHandler", e);
+ throw new RuntimeException("Failed to create SSLContext for Http2ProtocolProxyHandler", e);
+ }
+ }
+
+ @Override
+ public boolean match(ByteBuf in) {
+ if (!ConfigurationManager.getProxyConfig().isEnableRemotingLocalProxyGrpc()) {
+ return false;
+ }
+
+ // If starts with 'PRI '
+ return in.getInt(in.readerIndex()) == PRI_INT;
+ }
+
+ @Override
+ public void config(final ChannelHandlerContext ctx, final ByteBuf msg) {
+ // proxy channel to http2 server
+ final Channel inboundChannel = ctx.channel();
+
+ ProxyConfig config = ConfigurationManager.getProxyConfig();
+ // Start the connection attempt.
+ Bootstrap b = new Bootstrap();
+ b.group(inboundChannel.eventLoop())
+ .channel(ctx.channel().getClass())
+ .handler(new ChannelInitializer<Channel>() {
+ @Override
+ protected void initChannel(Channel ch) throws Exception {
+ if (sslContext != null) {
+ ch.pipeline()
+ .addLast(sslContext.newHandler(ch.alloc(), LOCAL_HOST, config.getGrpcServerPort()));
+ }
+ ch.pipeline().addLast(new Http2ProxyBackendHandler(inboundChannel));
+ }
+ })
+ .option(ChannelOption.AUTO_READ, false)
+ .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, config.getLocalProxyConnectTimeoutMs());
+ ChannelFuture f;
+ try {
+ f = b.connect(LOCAL_HOST, config.getGrpcServerPort()).sync();
+ } catch (Exception e) {
+ log.error("connect http2 server failed. port:{}", config.getGrpcServerPort(), e);
+ inboundChannel.close();
+ return;
+ }
+
+ final Channel outboundChannel = f.channel();
+
+ ctx.pipeline().addLast(new Http2ProxyFrontendHandler(outboundChannel));
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java
new file mode 100644
index 000000000..53bddfc31
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyBackendHandler.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.protocol.http2proxy;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+
+public class Http2ProxyBackendHandler extends ChannelInboundHandlerAdapter {
+ private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
+
+ private final Channel inboundChannel;
+
+ public Http2ProxyBackendHandler(Channel inboundChannel) {
+ this.inboundChannel = inboundChannel;
+ }
+
+ @Override
+ public void channelActive(ChannelHandlerContext ctx) {
+ ctx.read();
+ }
+
+ @Override
+ public void channelRead(final ChannelHandlerContext ctx, Object msg) {
+ inboundChannel.writeAndFlush(msg).addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ if (future.isSuccess()) {
+ ctx.channel().read();
+ } else {
+ future.channel().close();
+ }
+ }
+ });
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) {
+ Http2ProxyFrontendHandler.closeOnFlush(inboundChannel);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+ log.error("Http2ProxyBackendHandler#exceptionCaught", cause);
+ Http2ProxyFrontendHandler.closeOnFlush(ctx.channel());
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java
new file mode 100644
index 000000000..8bffdc6d0
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProxyFrontendHandler.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.protocol.http2proxy;
+
+import io.netty.buffer.Unpooled;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+
+public class Http2ProxyFrontendHandler extends ChannelInboundHandlerAdapter {
+ private static final InternalLogger log = InternalLoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING);
+ // As we use inboundChannel.eventLoop() when building the Bootstrap this does not need to be volatile as
+ // the outboundChannel will use the same EventLoop (and therefore Thread) as the inboundChannel.
+ private final Channel outboundChannel;
+
+ public Http2ProxyFrontendHandler(final Channel outboundChannel) {
+ this.outboundChannel = outboundChannel;
+ }
+
+ @Override
+ public void channelRead(final ChannelHandlerContext ctx, Object msg) {
+ if (outboundChannel.isActive()) {
+ outboundChannel.writeAndFlush(msg).addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ if (future.isSuccess()) {
+ // was able to flush out data, start to read the next chunk
+ ctx.channel().read();
+ } else {
+ future.channel().close();
+ }
+ }
+ });
+ }
+ }
+
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) {
+ if (outboundChannel != null) {
+ closeOnFlush(outboundChannel);
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+ log.error("Http2ProxyFrontendHandler#exceptionCaught", cause);
+ closeOnFlush(ctx.channel());
+ }
+
+ /**
+ * Closes the specified channel after all queued write requests are flushed.
+ */
+ static void closeOnFlush(Channel ch) {
+ if (ch.isActive()) {
+ ch.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE);
+ }
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java
new file mode 100644
index 000000000..3e4cc7c04
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.protocol.remoting;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import org.apache.rocketmq.proxy.remoting.protocol.ProtocolHandler;
+import org.apache.rocketmq.remoting.netty.NettyDecoder;
+import org.apache.rocketmq.remoting.netty.NettyEncoder;
+
+public class RemotingProtocolHandler implements ProtocolHandler {
+
+ private final NettyEncoder encoder;
+ private final ChannelDuplexHandler connectionManageHandler;
+ private final SimpleChannelInboundHandler serverHandler;
+
+ public RemotingProtocolHandler(NettyEncoder encoder, ChannelDuplexHandler connectionManageHandler,
+ SimpleChannelInboundHandler serverHandler) {
+ this.encoder = encoder;
+ this.connectionManageHandler = connectionManageHandler;
+ this.serverHandler = serverHandler;
+ }
+
+ @Override
+ public boolean match(ByteBuf in) {
+ return true;
+ }
+
+ @Override
+ public void config(ChannelHandlerContext ctx, ByteBuf msg) {
+ ctx.pipeline().addLast(
+ this.encoder,
+ new NettyDecoder(),
+ this.connectionManageHandler,
+ this.serverHandler
+ );
+ }
+}
[rocketmq] 08/26: [ISSUE #5486] polish MultiProtocolRemotingServer
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 4c0e06749bb664f27a9726b9a37827453f96fba8
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Wed Nov 9 16:44:03 2022 +0800
[ISSUE #5486] polish MultiProtocolRemotingServer
---
.../rocketmq/proxy/common/ReflectionCache.java | 45 ------------
.../remoting/MultiProtocolRemotingServer.java | 81 ++++------------------
.../http2proxy/Http2ProtocolProxyHandler.java | 2 +-
.../protocol/remoting/RemotingProtocolHandler.java | 17 +++--
.../remoting/netty/NettyRemotingServer.java | 68 +++++++++++++-----
5 files changed, 78 insertions(+), 135 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReflectionCache.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReflectionCache.java
deleted file mode 100644
index 31fa46c90..000000000
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReflectionCache.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.proxy.common;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import java.lang.reflect.Field;
-import java.util.concurrent.TimeUnit;
-
-public class ReflectionCache {
- private final Cache<Class<?>, Field> fieldCache;
- private static final int DEFAULT_MAX_SIZE = 15;
-
- public ReflectionCache() {
- this(DEFAULT_MAX_SIZE);
- }
-
- public ReflectionCache(int maxSize) {
- this.fieldCache = CacheBuilder.newBuilder().maximumSize(maxSize).expireAfterAccess(5, TimeUnit.MINUTES).build();
- }
-
- public Field getDeclaredField(final Class<?> clazz, final String fieldName) throws Exception {
- return this.fieldCache.get(clazz, () -> {
- Field field = clazz.getDeclaredField(fieldName);
- field.setAccessible(true);
- return field;
- });
- }
-}
-
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
index 73aeeaf42..02e3a545e 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolRemotingServer.java
@@ -17,18 +17,11 @@
package org.apache.rocketmq.proxy.remoting;
-import com.google.common.base.Preconditions;
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.ChannelDuplexHandler;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.channel.ChannelPipeline;
import io.netty.channel.socket.SocketChannel;
import io.netty.handler.timeout.IdleStateHandler;
-import io.netty.util.concurrent.DefaultEventExecutorGroup;
import java.io.IOException;
-import java.lang.reflect.Field;
import java.security.cert.CertificateException;
-import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.proxy.common.ProxyException;
import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
@@ -37,7 +30,6 @@ import org.apache.rocketmq.proxy.remoting.protocol.http2proxy.Http2ProtocolProxy
import org.apache.rocketmq.proxy.remoting.protocol.remoting.RemotingProtocolHandler;
import org.apache.rocketmq.remoting.ChannelEventListener;
import org.apache.rocketmq.remoting.common.TlsMode;
-import org.apache.rocketmq.remoting.netty.NettyEncoder;
import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
@@ -45,26 +37,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * for remoting server, if config listen port is 8080 in nettyServerConfig
- * <p>
- * will
- * <li>listen port at 9080 with protocol remoting</li>
- * <li>listen port at 8080 with protocol remoting and http2</li>
+ * support remoting and http2 protocol at one port
*/
public class MultiProtocolRemotingServer extends NettyRemotingServer {
private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
- private static final int PORT_DELTA = 1000;
private final NettyServerConfig nettyServerConfig;
- private final int port;
public MultiProtocolRemotingServer(NettyServerConfig nettyServerConfig, ChannelEventListener channelEventListener) {
super(nettyServerConfig, channelEventListener);
- this.port = nettyServerConfig.getListenPort();
- // to support multiple protocol
- // will bind the real port in configChildHandler
- // so let parent bind to a useless port
- nettyServerConfig.setListenPort(nettyServerConfig.getListenPort() + PORT_DELTA);
this.nettyServerConfig = nettyServerConfig;
}
@@ -84,50 +65,18 @@ public class MultiProtocolRemotingServer extends NettyRemotingServer {
}
@Override
- public void start() {
- super.start();
- this.configChildHandler();
- }
-
- protected void configChildHandler() {
- try {
- ServerBootstrap serverBootstrap = getField("serverBootstrap", ServerBootstrap.class);
- Preconditions.checkNotNull(serverBootstrap);
- DefaultEventExecutorGroup defaultEventExecutorGroup = getField("defaultEventExecutorGroup", DefaultEventExecutorGroup.class);
- Preconditions.checkNotNull(defaultEventExecutorGroup);
- NettyEncoder encoder = getField("encoder", NettyEncoder.class);
- Preconditions.checkNotNull(encoder);
- ChannelDuplexHandler connectionManageHandler = getField("connectionManageHandler", ChannelDuplexHandler.class);
- Preconditions.checkNotNull(connectionManageHandler);
- SimpleChannelInboundHandler serverHandler = getField("serverHandler", SimpleChannelInboundHandler.class);
- Preconditions.checkNotNull(serverHandler);
- SimpleChannelInboundHandler handshakeHandler = getField("handshakeHandler", SimpleChannelInboundHandler.class);
- Preconditions.checkNotNull(handshakeHandler);
- ConcurrentMap remotingServerTable = getField("remotingServerTable", ConcurrentMap.class);
- Preconditions.checkNotNull(remotingServerTable);
-
- serverBootstrap.childHandler(new ChannelInitializer<SocketChannel>() {
- @Override
- protected void initChannel(SocketChannel ch) {
- ch.pipeline()
- .addLast(defaultEventExecutorGroup, "handshakeHandler", handshakeHandler)
- .addLast(defaultEventExecutorGroup,
- new IdleStateHandler(0, 0, nettyServerConfig.getServerChannelMaxIdleTimeSeconds()),
- new ProtocolNegotiationHandler(new RemotingProtocolHandler(encoder, connectionManageHandler, serverHandler))
- .addProtocolHandler(new Http2ProtocolProxyHandler())
- );
- }
- });
- remotingServerTable.put(port, this);
- serverBootstrap.bind(port).sync();
- } catch (Throwable t) {
- throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "config netty child handler failed", t);
- }
- }
-
- protected <T> T getField(String name, Class<T> getClazz) throws Throwable {
- Field field = NettyRemotingServer.class.getDeclaredField(name);
- field.setAccessible(true);
- return getClazz.cast(field.get(this));
+ protected ChannelPipeline configChannel(SocketChannel ch) {
+ return ch.pipeline()
+ .addLast(this.getDefaultEventExecutorGroup(), HANDSHAKE_HANDLER_NAME, this.getHandshakeHandler())
+ .addLast(this.getDefaultEventExecutorGroup(),
+ new IdleStateHandler(0, 0, nettyServerConfig.getServerChannelMaxIdleTimeSeconds()),
+ new ProtocolNegotiationHandler(
+ new RemotingProtocolHandler(
+ this.getEncoder(),
+ this.getDistributionHandler(),
+ this.getConnectionManageHandler(),
+ this.getServerHandler()))
+ .addProtocolHandler(new Http2ProtocolProxyHandler())
+ );
}
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
index c5050cda7..86f1ee921 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
@@ -43,7 +43,7 @@ public class Http2ProtocolProxyHandler implements ProtocolHandler {
private static final String LOCAL_HOST = "127.0.0.1";
/**
* The int value of "PRI ". Now use 4 bytes to judge protocol, may be has potential risks if there is a new protocol
- * which start with "PRI " too in the future
+ * which start with "PRI " in the future
* <p>
* The full HTTP/2 connection preface is "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
* <p>
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java
index 3e4cc7c04..2d1a04d0e 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/remoting/RemotingProtocolHandler.java
@@ -18,22 +18,26 @@
package org.apache.rocketmq.proxy.remoting.protocol.remoting;
import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelDuplexHandler;
import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
import org.apache.rocketmq.proxy.remoting.protocol.ProtocolHandler;
import org.apache.rocketmq.remoting.netty.NettyDecoder;
import org.apache.rocketmq.remoting.netty.NettyEncoder;
+import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
+import org.apache.rocketmq.remoting.netty.RemotingCodeDistributionHandler;
public class RemotingProtocolHandler implements ProtocolHandler {
private final NettyEncoder encoder;
- private final ChannelDuplexHandler connectionManageHandler;
- private final SimpleChannelInboundHandler serverHandler;
+ private final RemotingCodeDistributionHandler remotingCodeDistributionHandler;
+ private final NettyRemotingServer.NettyConnectManageHandler connectionManageHandler;
+ private final NettyRemotingServer.NettyServerHandler serverHandler;
- public RemotingProtocolHandler(NettyEncoder encoder, ChannelDuplexHandler connectionManageHandler,
- SimpleChannelInboundHandler serverHandler) {
+ public RemotingProtocolHandler(NettyEncoder encoder,
+ RemotingCodeDistributionHandler remotingCodeDistributionHandler,
+ NettyRemotingServer.NettyConnectManageHandler connectionManageHandler,
+ NettyRemotingServer.NettyServerHandler serverHandler) {
this.encoder = encoder;
+ this.remotingCodeDistributionHandler = remotingCodeDistributionHandler;
this.connectionManageHandler = connectionManageHandler;
this.serverHandler = serverHandler;
}
@@ -48,6 +52,7 @@ public class RemotingProtocolHandler implements ProtocolHandler {
ctx.pipeline().addLast(
this.encoder,
new NettyDecoder(),
+ this.remotingCodeDistributionHandler,
this.connectionManageHandler,
this.serverHandler
);
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
index 1b364b6ee..646c0734e 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
@@ -26,6 +26,7 @@ import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.WriteBufferWaterMark;
@@ -93,9 +94,9 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
*/
private final ConcurrentMap<Integer/*Port*/, NettyRemotingAbstract> remotingServerTable = new ConcurrentHashMap<>();
- private static final String HANDSHAKE_HANDLER_NAME = "handshakeHandler";
- private static final String TLS_HANDLER_NAME = "sslHandler";
- private static final String FILE_REGION_ENCODER_NAME = "fileRegionEncoder";
+ public static final String HANDSHAKE_HANDLER_NAME = "handshakeHandler";
+ public static final String TLS_HANDLER_NAME = "sslHandler";
+ public static final String FILE_REGION_ENCODER_NAME = "fileRegionEncoder";
// sharable handlers
private HandshakeHandler handshakeHandler;
@@ -242,17 +243,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) {
- ch.pipeline()
- .addLast(defaultEventExecutorGroup, HANDSHAKE_HANDLER_NAME, handshakeHandler)
- .addLast(defaultEventExecutorGroup,
- encoder,
- new NettyDecoder(),
- distributionHandler,
- new IdleStateHandler(0, 0,
- nettyServerConfig.getServerChannelMaxIdleTimeSeconds()),
- connectionManageHandler,
- serverHandler
- );
+ configChannel(ch);
}
});
@@ -297,6 +288,25 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}, 1, 1, TimeUnit.SECONDS);
}
+ /**
+ * config channel in ChannelInitializer
+ * @param ch the SocketChannel needed to init
+ * @return the initialized ChannelPipeline, sub class can use it to extent in the future
+ */
+ protected ChannelPipeline configChannel(SocketChannel ch) {
+ return ch.pipeline()
+ .addLast(defaultEventExecutorGroup, HANDSHAKE_HANDLER_NAME, handshakeHandler)
+ .addLast(defaultEventExecutorGroup,
+ encoder,
+ new NettyDecoder(),
+ distributionHandler,
+ new IdleStateHandler(0, 0,
+ nettyServerConfig.getServerChannelMaxIdleTimeSeconds()),
+ connectionManageHandler,
+ serverHandler
+ );
+ }
+
private void addCustomConfig(ServerBootstrap childHandler) {
if (nettyServerConfig.getServerSocketSndBufSize() > 0) {
log.info("server set SO_SNDBUF to {}", nettyServerConfig.getServerSocketSndBufSize());
@@ -438,8 +448,32 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
}
+ public DefaultEventExecutorGroup getDefaultEventExecutorGroup() {
+ return defaultEventExecutorGroup;
+ }
+
+ public HandshakeHandler getHandshakeHandler() {
+ return handshakeHandler;
+ }
+
+ public NettyEncoder getEncoder() {
+ return encoder;
+ }
+
+ public NettyConnectManageHandler getConnectionManageHandler() {
+ return connectionManageHandler;
+ }
+
+ public NettyServerHandler getServerHandler() {
+ return serverHandler;
+ }
+
+ public RemotingCodeDistributionHandler getDistributionHandler() {
+ return distributionHandler;
+ }
+
@ChannelHandler.Sharable
- class HandshakeHandler extends SimpleChannelInboundHandler<ByteBuf> {
+ public class HandshakeHandler extends SimpleChannelInboundHandler<ByteBuf> {
private final TlsMode tlsMode;
@@ -496,7 +530,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
@ChannelHandler.Sharable
- class NettyServerHandler extends SimpleChannelInboundHandler<RemotingCommand> {
+ public class NettyServerHandler extends SimpleChannelInboundHandler<RemotingCommand> {
@Override
protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) {
@@ -529,7 +563,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
}
@ChannelHandler.Sharable
- class NettyConnectManageHandler extends ChannelDuplexHandler {
+ public class NettyConnectManageHandler extends ChannelDuplexHandler {
@Override
public void channelRegistered(ChannelHandlerContext ctx) throws Exception {
final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
[rocketmq] 10/26: [ISSUE #5486] polish tls config; add tcnative dependency
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 832fc57a08a1ee5aee18b662e1e7539e41caf50f
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Wed Nov 9 18:03:00 2022 +0800
[ISSUE #5486] polish tls config; add tcnative dependency
---
pom.xml | 6 +++
proxy/pom.xml | 4 ++
.../apache/rocketmq/proxy/config/ProxyConfig.java | 48 ++++++++++++----------
.../rocketmq/proxy/grpc/GrpcServerBuilder.java | 6 +--
.../proxy/remoting/RemotingProtocolServer.java | 12 +++---
5 files changed, 45 insertions(+), 31 deletions(-)
diff --git a/pom.xml b/pom.xml
index 197cea783..e4324a5b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,6 +103,7 @@
<commons-cli.version>1.5.0</commons-cli.version>
<netty.version>4.1.65.Final</netty.version>
+ <netty.tcnative.version>2.0.53.Final</netty.tcnative.version>
<bcpkix-jdk15on.version>1.69</bcpkix-jdk15on.version>
<fastjson.version>1.2.83</fastjson.version>
<javassist.version>3.20.0-GA</javassist.version>
@@ -900,6 +901,11 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>${netty.tcnative.version}</version>
+ </dependency>
<dependency>
<groupId>org.springframework</groupId>
diff --git a/proxy/pom.xml b/proxy/pom.xml
index c15734f16..f5373e914 100644
--- a/proxy/pom.xml
+++ b/proxy/pom.xml
@@ -95,6 +95,10 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ </dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
index b613c191e..bd7cf1113 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
@@ -73,14 +73,17 @@ public class ProxyConfig implements ConfigFile {
private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, System.getenv(MixAll.NAMESRV_ADDR_ENV));
private String namesrvDomain = "";
private String namesrvDomainSubgroup = "";
+ /**
+ * TLS
+ */
+ private boolean tlsTestModeEnable = true;
+ private String tlsKeyPath = ConfigurationManager.getProxyHome() + "/conf/tls/rocketmq.key";
+ private String tlsCertPath = ConfigurationManager.getProxyHome() + "/conf/tls/rocketmq.crt";
/**
* gRPC
*/
private String proxyMode = ProxyMode.CLUSTER.name();
private Integer grpcServerPort = 8081;
- private boolean grpcTlsTestModeEnable = true;
- private String grpcTlsKeyPath = ConfigurationManager.getProxyHome() + "/conf/tls/rocketmq.key";
- private String grpcTlsCertPath = ConfigurationManager.getProxyHome() + "/conf/tls/rocketmq.crt";
private int grpcBossLoopNum = 1;
private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2;
private boolean enableGrpcEpoll = false;
@@ -190,8 +193,6 @@ public class ProxyConfig implements ConfigFile {
private boolean traceOn = false;
- private String remotingAccessPoint = "";
-
private BrokerConfig.MetricsExporterType metricsExporterType = BrokerConfig.MetricsExporterType.DISABLE;
private String metricsGrpcExporterTarget = "";
@@ -210,9 +211,9 @@ public class ProxyConfig implements ConfigFile {
private long channelExpiredTimeout = 1000 * 120;
// remoting
-
private boolean enableRemotingLocalProxyGrpc = true;
private int localProxyConnectTimeoutMs = 3000;
+ private String remotingAccessAddr = "";
private int remotingListenPort = 8080;
private int remotingHeartbeatThreadPoolNums = 2 * PROCESSOR_NUMBER;
@@ -245,6 +246,9 @@ public class ProxyConfig implements ConfigFile {
if (StringUtils.isBlank(localServeAddr)) {
throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "get local serve ip failed");
}
+ if (StringUtils.isBlank(remotingAccessAddr)) {
+ this.remotingAccessAddr = this.localServeAddr;
+ }
if (StringUtils.isBlank(systemTopicClusterName)) {
this.systemTopicClusterName = this.rocketMQClusterName;
}
@@ -407,28 +411,28 @@ public class ProxyConfig implements ConfigFile {
this.grpcServerPort = grpcServerPort;
}
- public boolean isGrpcTlsTestModeEnable() {
- return grpcTlsTestModeEnable;
+ public boolean isTlsTestModeEnable() {
+ return tlsTestModeEnable;
}
- public void setGrpcTlsTestModeEnable(boolean grpcTlsTestModeEnable) {
- this.grpcTlsTestModeEnable = grpcTlsTestModeEnable;
+ public void setTlsTestModeEnable(boolean tlsTestModeEnable) {
+ this.tlsTestModeEnable = tlsTestModeEnable;
}
- public String getGrpcTlsKeyPath() {
- return grpcTlsKeyPath;
+ public String getTlsKeyPath() {
+ return tlsKeyPath;
}
- public void setGrpcTlsKeyPath(String grpcTlsKeyPath) {
- this.grpcTlsKeyPath = grpcTlsKeyPath;
+ public void setTlsKeyPath(String tlsKeyPath) {
+ this.tlsKeyPath = tlsKeyPath;
}
- public String getGrpcTlsCertPath() {
- return grpcTlsCertPath;
+ public String getTlsCertPath() {
+ return tlsCertPath;
}
- public void setGrpcTlsCertPath(String grpcTlsCertPath) {
- this.grpcTlsCertPath = grpcTlsCertPath;
+ public void setTlsCertPath(String tlsCertPath) {
+ this.tlsCertPath = tlsCertPath;
}
public int getGrpcBossLoopNum() {
@@ -1059,12 +1063,12 @@ public class ProxyConfig implements ConfigFile {
this.traceOn = traceOn;
}
- public String getRemotingAccessPoint() {
- return remotingAccessPoint;
+ public String getRemotingAccessAddr() {
+ return remotingAccessAddr;
}
- public void setRemotingAccessPoint(String remotingAccessPoint) {
- this.remotingAccessPoint = remotingAccessPoint;
+ public void setRemotingAccessAddr(String remotingAccessAddr) {
+ this.remotingAccessAddr = remotingAccessAddr;
}
public BrokerConfig.MetricsExporterType getMetricsExporterType() {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
index 00a738770..5e1b73505 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
@@ -118,7 +118,7 @@ public class GrpcServerBuilder {
return;
}
ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
- boolean tlsTestModeEnable = proxyConfig.isGrpcTlsTestModeEnable();
+ boolean tlsTestModeEnable = proxyConfig.isTlsTestModeEnable();
if (tlsTestModeEnable) {
SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate();
serverBuilder.sslContext(GrpcSslContexts.forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey())
@@ -128,8 +128,8 @@ public class GrpcServerBuilder {
return;
}
- String tlsKeyPath = ConfigurationManager.getProxyConfig().getGrpcTlsKeyPath();
- String tlsCertPath = ConfigurationManager.getProxyConfig().getGrpcTlsCertPath();
+ String tlsKeyPath = ConfigurationManager.getProxyConfig().getTlsKeyPath();
+ String tlsCertPath = ConfigurationManager.getProxyConfig().getTlsCertPath();
try (InputStream serverKeyInputStream = Files.newInputStream(Paths.get(tlsKeyPath));
InputStream serverCertificateStream = Files.newInputStream(Paths.get(tlsCertPath))) {
serverBuilder.sslContext(GrpcSslContexts.forServer(serverCertificateStream, serverKeyInputStream)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
index d0137b2b4..a173a79b6 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -99,12 +99,12 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
ProxyConfig config = ConfigurationManager.getProxyConfig();
NettyServerConfig defaultServerConfig = new NettyServerConfig();
defaultServerConfig.setListenPort(config.getRemotingListenPort());
- TlsSystemConfig.tlsTestModeEnable = false;
- System.setProperty(TlsSystemConfig.TLS_TEST_MODE_ENABLE, "false");
- TlsSystemConfig.tlsServerCertPath = config.getGrpcTlsCertPath();
- System.setProperty(TlsSystemConfig.TLS_SERVER_CERTPATH, config.getGrpcTlsCertPath());
- TlsSystemConfig.tlsServerKeyPath = config.getGrpcTlsKeyPath();
- System.setProperty(TlsSystemConfig.TLS_SERVER_KEYPATH, config.getGrpcTlsKeyPath());
+ TlsSystemConfig.tlsTestModeEnable = config.isTlsTestModeEnable();
+ System.setProperty(TlsSystemConfig.TLS_TEST_MODE_ENABLE, Boolean.toString(config.isTlsTestModeEnable()));
+ TlsSystemConfig.tlsServerCertPath = config.getTlsCertPath();
+ System.setProperty(TlsSystemConfig.TLS_SERVER_CERTPATH, config.getTlsCertPath());
+ TlsSystemConfig.tlsServerKeyPath = config.getTlsKeyPath();
+ System.setProperty(TlsSystemConfig.TLS_SERVER_KEYPATH, config.getTlsKeyPath());
this.clientHousekeepingService = new ClientHousekeepingService(this.clientManagerActivity);
[rocketmq] 07/26: [ISSUE #5486] polish clean expire request in remoting server
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 8a23c54a9193a6fd76446a5d6b8b222d3bdaed77
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Wed Nov 9 15:43:52 2022 +0800
[ISSUE #5486] polish clean expire request in remoting server
---
.../thread/FutureTaskExtThreadPoolExecutor.java | 41 ++++++++++++++++++++++
.../rocketmq/common/thread/ThreadPoolMonitor.java | 2 +-
.../proxy/remoting/RemotingProtocolServer.java | 33 ++++++-----------
3 files changed, 52 insertions(+), 24 deletions(-)
diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java
new file mode 100644
index 000000000..411da9221
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.thread;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.future.FutureTaskExt;
+
+public class FutureTaskExtThreadPoolExecutor extends ThreadPoolExecutor {
+
+ public FutureTaskExtThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime,
+ TimeUnit unit,
+ BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory,
+ RejectedExecutionHandler handler) {
+ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
+ }
+
+ @Override
+ protected <T> RunnableFuture<T> newTaskFor(final Runnable runnable, final T value) {
+ return new FutureTaskExt<>(runnable, value);
+ }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
index 72d4384c4..49d97a5d7 100644
--- a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
+++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
@@ -81,7 +81,7 @@ public class ThreadPoolMonitor {
String name,
int queueCapacity,
List<ThreadPoolStatusMonitor> threadPoolStatusMonitors) {
- ThreadPoolExecutor executor = new ThreadPoolExecutor(
+ ThreadPoolExecutor executor = new FutureTaskExtThreadPoolExecutor(
corePoolSize,
maximumPoolSize,
keepAliveTime,
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
index fdf1870a5..91c4422d2 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -17,22 +17,21 @@
package org.apache.rocketmq.proxy.remoting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import io.netty.channel.Channel;
-import java.lang.reflect.Field;
import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.FutureTask;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import org.apache.rocketmq.broker.latency.FutureTaskExt;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.future.FutureTaskExt;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
import org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor;
-import org.apache.rocketmq.proxy.common.ReflectionCache;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.config.ProxyConfig;
@@ -78,8 +77,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
protected final ThreadPoolExecutor updateOffsetExecutor;
protected final ThreadPoolExecutor topicRouteExecutor;
protected final ThreadPoolExecutor defaultExecutor;
-
- private final ReflectionCache reflectionCache = new ReflectionCache();
+ protected final ScheduledExecutorService timerExecutor;
public RemotingProtocolServer(MessagingProcessor messagingProcessor) {
this.messagingProcessor = messagingProcessor;
@@ -173,10 +171,11 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
config.getRemotingDefaultThreadPoolQueueCapacity(),
new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInDefaultQueue())
);
- }
-
- protected void init() {
+ this.timerExecutor = Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactoryBuilder().setNameFormat("RemotingServerScheduler-%d").build()
+ );
+ this.timerExecutor.scheduleAtFixedRate(this::cleanExpireRequest, 10, 10, TimeUnit.SECONDS);
}
protected void registerRemotingServer(RemotingServer remotingServer) {
@@ -342,19 +341,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
private RequestTask castRunnable(final Runnable runnable) {
try {
- if (runnable instanceof FutureTask) {
- Field callableField = reflectionCache.getDeclaredField(FutureTask.class, "callable");
- Callable callable = (Callable) callableField.get(runnable);
- if (callable == null) {
- return null;
- }
- Field taskField = reflectionCache.getDeclaredField(callable.getClass(), "task");
- if (taskField == null) {
- log.warn("get task from FutureTask failed. class:{}", runnable.getClass().getName());
- return null;
- }
- return (RequestTask) taskField.get(callable);
- } else if (runnable instanceof FutureTaskExt) {
+ if (runnable instanceof FutureTaskExt) {
FutureTaskExt futureTaskExt = (FutureTaskExt) runnable;
return (RequestTask) futureTaskExt.getRunnable();
}
[rocketmq] 01/26: [ISSUE #5406] Add processor for remoting messaging module
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 934f5b657639aede31a70aa902b25207ba16d179
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Thu Oct 27 16:00:13 2022 +0800
[ISSUE #5406] Add processor for remoting messaging module
---
.../common/attribute/TopicMessageType.java | 18 +++
.../apache/rocketmq/proxy/config/ProxyConfig.java | 30 ++++
.../proxy/processor/AbstractProcessor.java | 19 ---
.../proxy/processor/DefaultMessagingProcessor.java | 14 ++
.../proxy/processor/MessagingProcessor.java | 6 +
.../proxy/processor/ProducerProcessor.java | 2 +-
.../proxy/processor/RequestBrokerProcessor.java | 39 +++++
.../activity/AbstractRemotingActivity.java | 166 +++++++++++++++++++++
.../remoting/activity/AckMessageActivity.java | 38 +++++
.../activity/ChangeInvisibleTimeActivity.java | 38 +++++
.../remoting/activity/ConsumerManagerActivity.java | 112 ++++++++++++++
.../remoting/activity/GetTopicRouteActivity.java | 71 +++++++++
.../remoting/activity/PopMessageActivity.java | 41 +++++
.../remoting/activity/PullMessageActivity.java | 72 +++++++++
.../remoting/activity/SendMessageActivity.java | 96 ++++++++++++
.../proxy/remoting/pipeline/RequestPipeline.java | 34 ++---
.../proxy/service/channel/SimpleChannel.java | 5 +
.../service/message/ClusterMessageService.java | 37 ++++-
.../proxy/service/message/LocalMessageService.java | 12 ++
.../proxy/service/message/MessageService.java | 6 +
.../activity/AbstractRemotingActivityTest.java | 84 +++++++++++
21 files changed, 892 insertions(+), 48 deletions(-)
diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java b/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
index 8c484da31..5e6629e3b 100644
--- a/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
+++ b/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
@@ -18,7 +18,9 @@
package org.apache.rocketmq.common.attribute;
import com.google.common.collect.Sets;
+import java.util.Map;
import java.util.Set;
+import org.apache.rocketmq.common.message.MessageConst;
public enum TopicMessageType {
UNSPECIFIED("UNSPECIFIED"),
@@ -40,6 +42,22 @@ public enum TopicMessageType {
return value;
}
+ public static TopicMessageType parseFromMessageProperty(Map<String, String> messageProperty) {
+ String isTrans = messageProperty.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
+ String isTransValue = "true";
+ if (isTransValue.equals(isTrans)) {
+ return TopicMessageType.TRANSACTION;
+ } else if (messageProperty.get(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null
+ || messageProperty.get(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null
+ || messageProperty.get(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) {
+ return TopicMessageType.DELAY;
+ } else if (messageProperty.get(MessageConst.PROPERTY_SHARDING_KEY) != null) {
+ return TopicMessageType.FIFO;
+ } else {
+ return TopicMessageType.NORMAL;
+ }
+ }
+
public String getMetricsValue() {
return value.toLowerCase();
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
index 6bb488984..cbedc3c50 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
@@ -178,6 +178,12 @@ public class ProxyConfig implements ConfigFile {
// Example address: 127.0.0.1:1234
private String metricCollectorAddress = "";
+ private String regionId = "";
+
+ private boolean traceOn = false;
+
+ private String remotingAccessPoint = "";
+
private BrokerConfig.MetricsExporterType metricsExporterType = BrokerConfig.MetricsExporterType.DISABLE;
private String metricsGrpcExporterTarget = "";
@@ -961,6 +967,30 @@ public class ProxyConfig implements ConfigFile {
this.grpcClientIdleTimeMills = grpcClientIdleTimeMills;
}
+ public String getRegionId() {
+ return regionId;
+ }
+
+ public void setRegionId(String regionId) {
+ this.regionId = regionId;
+ }
+
+ public boolean isTraceOn() {
+ return traceOn;
+ }
+
+ public void setTraceOn(boolean traceOn) {
+ this.traceOn = traceOn;
+ }
+
+ public String getRemotingAccessPoint() {
+ return remotingAccessPoint;
+ }
+
+ public void setRemotingAccessPoint(String remotingAccessPoint) {
+ this.remotingAccessPoint = remotingAccessPoint;
+ }
+
public BrokerConfig.MetricsExporterType getMetricsExporterType() {
return metricsExporterType;
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
index c223eb478..679cc4b3d 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java
@@ -16,10 +16,7 @@
*/
package org.apache.rocketmq.proxy.processor;
-import org.apache.rocketmq.common.attribute.TopicMessageType;
import org.apache.rocketmq.common.consumer.ReceiptHandle;
-import org.apache.rocketmq.common.message.Message;
-import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown;
import org.apache.rocketmq.proxy.common.ProxyException;
import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
@@ -41,20 +38,4 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown {
throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired");
}
}
-
- protected TopicMessageType parseFromMessageExt(Message message) {
- String isTrans = message.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
- String isTransValue = "true";
- if (isTransValue.equals(isTrans)) {
- return TopicMessageType.TRANSACTION;
- } else if (message.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null
- || message.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null
- || message.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) {
- return TopicMessageType.DELAY;
- } else if (message.getProperty(MessageConst.PROPERTY_SHARDING_KEY) != null) {
- return TopicMessageType.FIFO;
- } else {
- return TopicMessageType.NORMAL;
- }
- }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
index 95fba895a..1b7baba0a 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
@@ -60,6 +60,7 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen
protected ConsumerProcessor consumerProcessor;
protected TransactionProcessor transactionProcessor;
protected ClientProcessor clientProcessor;
+ protected RequestBrokerProcessor requestBrokerProcessor;
protected ThreadPoolExecutor producerProcessorExecutor;
protected ThreadPoolExecutor consumerProcessorExecutor;
@@ -88,6 +89,7 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen
this.consumerProcessor = new ConsumerProcessor(this, serviceManager, this.consumerProcessorExecutor);
this.transactionProcessor = new TransactionProcessor(this, serviceManager);
this.clientProcessor = new ClientProcessor(this, serviceManager);
+ this.requestBrokerProcessor = new RequestBrokerProcessor(this, serviceManager);
this.init();
}
@@ -218,6 +220,18 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen
return this.consumerProcessor.getMinOffset(ctx, messageQueue, timeoutMillis);
}
+ @Override
+ public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis) {
+ return this.requestBrokerProcessor.request(ctx, brokerName, request, timeoutMillis);
+ }
+
+ @Override
+ public CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis) {
+ return this.requestBrokerProcessor.requestOneway(ctx, brokerName, request, timeoutMillis);
+ }
+
@Override
public void registerProducer(ProxyContext ctx, String producerGroup, ClientChannelInfo clientChannelInfo) {
this.clientProcessor.registerProducer(ctx, producerGroup, clientChannelInfo);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
index 89be595ec..3e8b8084e 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
@@ -230,6 +230,12 @@ public interface MessagingProcessor extends StartAndShutdown {
long timeoutMillis
);
+ CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis);
+
+ CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis);
+
void registerProducer(
ProxyContext ctx,
String producerGroup,
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
index 95bd0e5fe..2fce78d31 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
@@ -73,7 +73,7 @@ public class ProducerProcessor extends AbstractProcessor {
// Do not check retry or dlq topic
if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {
TopicMessageType topicMessageType = serviceManager.getMetadataService().getTopicMessageType(topic);
- TopicMessageType messageType = parseFromMessageExt(message);
+ TopicMessageType messageType = TopicMessageType.parseFromMessageProperty(message.getProperties());
topicMessageTypeValidator.validate(topicMessageType, messageType);
}
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/RequestBrokerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/RequestBrokerProcessor.java
new file mode 100644
index 000000000..9f3187cde
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/RequestBrokerProcessor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.processor;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.service.ServiceManager;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class RequestBrokerProcessor extends AbstractProcessor {
+
+ public RequestBrokerProcessor(MessagingProcessor messagingProcessor,
+ ServiceManager serviceManager) {
+ super(messagingProcessor, serviceManager);
+ }
+
+ CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis) {
+ return serviceManager.getMessageService().request(ctx, brokerName, request, timeoutMillis);
+ }
+
+ CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis) {
+ return serviceManager.getMessageService().requestOneway(ctx, brokerName, request, timeoutMillis);
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
new file mode 100644
index 000000000..7f0d891ec
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.common.ProxyException;
+import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
+import org.apache.rocketmq.proxy.common.utils.ExceptionUtils;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class AbstractRemotingActivity implements NettyRequestProcessor {
+ protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+ protected final MessagingProcessor messagingProcessor;
+ protected static final String BROKER_NAME_FIELD = "bname";
+ private static final Map<ProxyExceptionCode, Integer> PROXY_EXCEPTION_RESPONSE_CODE_MAP = new HashMap<ProxyExceptionCode, Integer>() {
+ {
+ put(ProxyExceptionCode.FORBIDDEN, ResponseCode.NO_PERMISSION);
+ put(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, ResponseCode.MESSAGE_ILLEGAL);
+ put(ProxyExceptionCode.INTERNAL_SERVER_ERROR, ResponseCode.SYSTEM_ERROR);
+ put(ProxyExceptionCode.TRANSACTION_DATA_NOT_FOUND, ResponseCode.SUCCESS);
+ }
+ };
+ protected final RequestPipeline requestPipeline;
+
+ public AbstractRemotingActivity(RequestPipeline requestPipeline, MessagingProcessor messagingProcessor) {
+ this.requestPipeline = requestPipeline;
+ this.messagingProcessor = messagingProcessor;
+ }
+
+ protected RemotingCommand request(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context, long timeoutMillis) throws Exception {
+ if (request.getExtFields().get(BROKER_NAME_FIELD) == null) {
+ return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,
+ "Request doesn't have field bname");
+ }
+ String brokerName = request.getExtFields().get(BROKER_NAME_FIELD);
+ if (request.isOnewayRPC()) {
+ return null;
+ }
+ messagingProcessor.request(context, brokerName, request, timeoutMillis)
+ .thenAccept(r -> writeResponse(ctx, context, request, r))
+ .exceptionally(t -> {
+ writeErrResponse(ctx, context, request, t);
+ return null;
+ });
+ return null;
+ }
+
+ @Override
+ public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {
+ ProxyContext context = createContext(ctx, request);
+ try {
+ this.requestPipeline.execute(ctx, request, context);
+ RemotingCommand response = this.processRequest0(ctx, request, context);
+ if (response != null) {
+ writeResponse(ctx, context, request, response);
+ }
+ return null;
+ } catch (Throwable t) {
+ writeErrResponse(ctx, context, request, t);
+ return null;
+ }
+ }
+
+ @Override
+ public boolean rejectRequest() {
+ return false;
+ }
+
+ protected abstract RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception;
+
+ protected ProxyContext createContext(ChannelHandlerContext ctx, RemotingCommand request) {
+ ProxyContext context = ProxyContext.create();
+ context.setAction("Remoting" + request.getCode())
+ .setLanguage(request.getLanguage().name())
+ .setChannel(ctx.channel())
+ .setLocalAddress(RemotingUtil.socketAddress2String(ctx.channel().localAddress()))
+ .setRemoteAddress(RemotingUtil.socketAddress2String(ctx.channel().remoteAddress()));
+
+ return context;
+ }
+
+ protected void writeErrResponse(ChannelHandlerContext ctx, final ProxyContext context,
+ final RemotingCommand request, Throwable t) {
+ t = ExceptionUtils.getRealException(t);
+ if (t instanceof ProxyException) {
+ ProxyException e = (ProxyException) t;
+ writeResponse(ctx, context, request,
+ RemotingCommand.createResponseCommand(
+ PROXY_EXCEPTION_RESPONSE_CODE_MAP.getOrDefault(e.getCode(), ResponseCode.SYSTEM_ERROR),
+ e.getMessage()),
+ t);
+ } else if (t instanceof MQClientException) {
+ MQClientException e = (MQClientException) t;
+ writeResponse(ctx, context, request, RemotingCommand.createResponseCommand(e.getResponseCode(), e.getErrorMessage()), t);
+ } else if (t instanceof MQBrokerException) {
+ MQBrokerException e = (MQBrokerException) t;
+ writeResponse(ctx, context, request, RemotingCommand.createResponseCommand(e.getResponseCode(), e.getErrorMessage()), t);
+ } else if (t instanceof AclException) {
+ writeResponse(ctx, context, request, RemotingCommand.createResponseCommand(ResponseCode.NO_PERMISSION, t.getMessage()), t);
+ } else {
+ writeResponse(ctx, context, request,
+ RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, t.getMessage()), t);
+ }
+ }
+
+ protected void writeResponse(ChannelHandlerContext ctx, final ProxyContext context,
+ final RemotingCommand request, RemotingCommand response) {
+ writeResponse(ctx, context, request, response, null);
+ }
+
+ protected void writeResponse(ChannelHandlerContext ctx, final ProxyContext context,
+ final RemotingCommand request, RemotingCommand response, Throwable t) {
+ if (request.isOnewayRPC()) {
+ return;
+ }
+ if (!ctx.channel().isWritable()) {
+ return;
+ }
+
+ ProxyConfig config = ConfigurationManager.getProxyConfig();
+
+ response.setOpaque(request.getOpaque());
+ response.markResponseType();
+ response.addExtField(MessageConst.PROPERTY_MSG_REGION, config.getRegionId());
+ response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(config.isTraceOn()));
+ if (t != null) {
+ response.setRemark(t.getMessage());
+ }
+
+ ctx.writeAndFlush(response);
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AckMessageActivity.java
new file mode 100644
index 000000000..723b5918b
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AckMessageActivity.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.time.Duration;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class AckMessageActivity extends AbstractRemotingActivity {
+ public AckMessageActivity(RequestPipeline requestPipeline,
+ MessagingProcessor messagingProcessor) {
+ super(requestPipeline, messagingProcessor);
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ return request(ctx, request, context, Duration.ofSeconds(3).toMillis());
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ChangeInvisibleTimeActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ChangeInvisibleTimeActivity.java
new file mode 100644
index 000000000..9f6de99e0
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ChangeInvisibleTimeActivity.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.time.Duration;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class ChangeInvisibleTimeActivity extends AbstractRemotingActivity {
+ public ChangeInvisibleTimeActivity(RequestPipeline requestPipeline,
+ MessagingProcessor messagingProcessor) {
+ super(requestPipeline, messagingProcessor);
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ return request(ctx, request, context, Duration.ofSeconds(3).toMillis());
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
new file mode 100644
index 000000000..fb248a894
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Set;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
+import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class ConsumerManagerActivity extends AbstractRemotingActivity {
+ public ConsumerManagerActivity(RequestPipeline requestPipeline, MessagingProcessor messagingProcessor) {
+ super(requestPipeline, messagingProcessor);
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ switch (request.getCode()) {
+ case RequestCode.GET_CONSUMER_LIST_BY_GROUP: {
+ return getConsumerListByGroup(ctx, request, context);
+ }
+ case RequestCode.LOCK_BATCH_MQ: {
+ return lockBatchMQ(ctx, request, context);
+ }
+ case RequestCode.UNLOCK_BATCH_MQ: {
+ return unlockBatchMQ(ctx, request, context);
+ }
+ case RequestCode.UPDATE_CONSUMER_OFFSET:
+ case RequestCode.QUERY_CONSUMER_OFFSET:
+ case RequestCode.SEARCH_OFFSET_BY_TIMESTAMP:
+ case RequestCode.GET_MIN_OFFSET:
+ case RequestCode.GET_MAX_OFFSET:
+ case RequestCode.GET_EARLIEST_MSG_STORETIME: {
+ return request(ctx, request, context, Duration.ofSeconds(3).toMillis());
+ }
+ default:
+ break;
+ }
+ return null;
+ }
+
+ protected RemotingCommand getConsumerListByGroup(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ // TODO after connection-related module
+ return null;
+ }
+
+ protected RemotingCommand lockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ LockBatchRequestBody requestBody = LockBatchRequestBody.decode(request.getBody(), LockBatchRequestBody.class);
+ Set<MessageQueue> mqSet = requestBody.getMqSet();
+ if (mqSet.isEmpty()) {
+ response.setBody(requestBody.encode());
+ response.setRemark("MessageQueue set is empty");
+ return response;
+ }
+
+ String brokerName = new ArrayList<>(mqSet).get(0).getBrokerName();
+ messagingProcessor.request(context, brokerName, request, Duration.ofSeconds(3).toMillis())
+ .thenAccept(r -> writeResponse(ctx, context, request, r))
+ .exceptionally(t -> {
+ writeErrResponse(ctx, context, request, t);
+ return null;
+ });
+ return null;
+ }
+
+ protected RemotingCommand unlockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ UnlockBatchRequestBody requestBody = UnlockBatchRequestBody.decode(request.getBody(), UnlockBatchRequestBody.class);
+ Set<MessageQueue> mqSet = requestBody.getMqSet();
+ if (mqSet.isEmpty()) {
+ response.setBody(requestBody.encode());
+ response.setRemark("MessageQueue set is empty");
+ return response;
+ }
+
+ String brokerName = new ArrayList<>(mqSet).get(0).getBrokerName();
+ messagingProcessor.request(context, brokerName, request, Duration.ofSeconds(3).toMillis())
+ .thenAccept(r -> writeResponse(ctx, context, request, r))
+ .exceptionally(t -> {
+ writeErrResponse(ctx, context, request, t);
+ return null;
+ });
+ return null;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
new file mode 100644
index 000000000..d3b7de98d
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.google.common.net.HostAndPort;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.rocketmq.common.MQVersion;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.proxy.common.Address;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class GetTopicRouteActivity extends AbstractRemotingActivity {
+ public GetTopicRouteActivity(RequestPipeline requestPipeline,
+ MessagingProcessor messagingProcessor) {
+ super(requestPipeline, messagingProcessor);
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ final GetRouteInfoRequestHeader requestHeader =
+ (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);
+ List<Address> addressList = new ArrayList<>();
+ addressList.add(new Address(Address.AddressScheme.IPv4, HostAndPort.fromString(proxyConfig.getRemotingAccessPoint())));
+ ProxyTopicRouteData proxyTopicRouteData = messagingProcessor.getTopicRouteDataForProxy(context, addressList, requestHeader.getTopic());
+ TopicRouteData topicRouteData = proxyTopicRouteData.buildTopicRouteData();
+
+ byte[] content;
+ Boolean standardJsonOnly = requestHeader.getAcceptStandardJsonOnly();
+ if (request.getVersion() >= MQVersion.Version.V4_9_4.ordinal() || null != standardJsonOnly && standardJsonOnly) {
+ content = topicRouteData.encode(SerializerFeature.BrowserCompatible,
+ SerializerFeature.QuoteFieldNames, SerializerFeature.SkipTransientField,
+ SerializerFeature.MapSortField);
+ } else {
+ content = topicRouteData.encode();
+ }
+
+ response.setBody(content);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java
new file mode 100644
index 000000000..d52b84b12
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PopMessageActivity.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.time.Duration;
+import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class PopMessageActivity extends AbstractRemotingActivity {
+ public PopMessageActivity(RequestPipeline requestPipeline,
+ MessagingProcessor messagingProcessor) {
+ super(requestPipeline, messagingProcessor);
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ PopMessageRequestHeader popMessageRequestHeader = (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class);
+ long timeoutMillis = popMessageRequestHeader.getPollTime();
+ return request(ctx, request, context, timeoutMillis + Duration.ofSeconds(10).toMillis());
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
new file mode 100644
index 000000000..819bf139d
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.common.sysflag.PullSysFlag;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class PullMessageActivity extends AbstractRemotingActivity {
+ public PullMessageActivity(RequestPipeline requestPipeline,
+ MessagingProcessor messagingProcessor) {
+ super(requestPipeline, messagingProcessor);
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ if (request.getExtFields().get(BROKER_NAME_FIELD) == null) {
+ return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,
+ "Request doesn't have field bname");
+ }
+ PullMessageRequestHeader requestHeader = (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class);
+ if (!PullSysFlag.hasSubscriptionFlag(requestHeader.getSysFlag())) {
+ ConsumerGroupInfo consumerInfo = messagingProcessor.getConsumerGroupInfo(requestHeader.getConsumerGroup());
+ if (consumerInfo == null) {
+ return RemotingCommand.buildErrorResponse(ResponseCode.SUBSCRIPTION_NOT_LATEST,
+ "the consumer's subscription not latest");
+ }
+ SubscriptionData subscriptionData = consumerInfo.findSubscriptionData(requestHeader.getTopic());
+ if (subscriptionData == null) {
+ return RemotingCommand.buildErrorResponse(ResponseCode.SUBSCRIPTION_NOT_EXIST,
+ "the consumer's subscription not exist");
+ }
+ requestHeader.setSubscription(subscriptionData.getSubString());
+ requestHeader.setExpressionType(subscriptionData.getExpressionType());
+ request.makeCustomHeaderToNet();
+ }
+ String brokerName = requestHeader.getBname();
+ long timeoutMillis = requestHeader.getSuspendTimeoutMillis() + Duration.ofSeconds(10).toMillis();
+ CompletableFuture<RemotingCommand> future = messagingProcessor.request(context, brokerName, request, timeoutMillis);
+ future.thenAccept(r -> writeResponse(ctx, context, request, r))
+ .exceptionally(t -> {
+ writeErrResponse(ctx, context, request, t);
+ return null;
+ });
+ return null;
+ }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
new file mode 100644
index 000000000..904460431
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.time.Duration;
+import java.util.Map;
+import org.apache.rocketmq.common.attribute.TopicMessageType;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.NamespaceUtil;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.processor.validator.DefaultTopicMessageTypeValidator;
+import org.apache.rocketmq.proxy.processor.validator.TopicMessageTypeValidator;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class SendMessageActivity extends AbstractRemotingActivity {
+ TopicMessageTypeValidator topicMessageTypeValidator;
+
+ public SendMessageActivity(RequestPipeline requestPipeline,
+ MessagingProcessor messagingProcessor) {
+ super(requestPipeline, messagingProcessor);
+ this.topicMessageTypeValidator = new DefaultTopicMessageTypeValidator();
+ }
+
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ switch (request.getCode()) {
+ case RequestCode.SEND_MESSAGE:
+ case RequestCode.SEND_MESSAGE_V2:
+ case RequestCode.SEND_BATCH_MESSAGE: {
+ return sendMessage(ctx, request, context);
+ }
+ case RequestCode.CONSUMER_SEND_MSG_BACK: {
+ return consumerSendMessage(ctx, request, context);
+ }
+ default:
+ break;
+ }
+ return null;
+ }
+
+ protected RemotingCommand sendMessage(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ SendMessageRequestHeader requestHeader = SendMessageRequestHeader.parseRequestHeader(request);
+ String topic = requestHeader.getTopic();
+ Map<String, String> property = MessageDecoder.string2messageProperties(requestHeader.getProperties());
+ TopicMessageType messageType = TopicMessageType.parseFromMessageProperty(property);
+ if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) {
+ if (topicMessageTypeValidator != null) {
+ // Do not check retry or dlq topic
+ if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {
+ TopicMessageType topicMessageType = messagingProcessor.getMetadataService().getTopicMessageType(topic);
+ topicMessageTypeValidator.validate(topicMessageType, messageType);
+ }
+ }
+ }
+ if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {
+ if (TopicMessageType.TRANSACTION.equals(messageType)) {
+ return sendTransactionMessage(ctx, request, context);
+ }
+ }
+ return request(ctx, request, context, Duration.ofSeconds(3).toMillis());
+ }
+
+ protected RemotingCommand consumerSendMessage(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ return request(ctx, request, context, Duration.ofSeconds(3).toMillis());
+ }
+
+ protected RemotingCommand sendTransactionMessage(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ // TODO: wait for connection implement.
+ return null;
+ }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/RequestPipeline.java
similarity index 54%
copy from common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
copy to proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/RequestPipeline.java
index 8c484da31..4c46a6e7d 100644
--- a/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/RequestPipeline.java
@@ -15,32 +15,20 @@
* limitations under the License.
*/
-package org.apache.rocketmq.common.attribute;
+package org.apache.rocketmq.proxy.remoting.pipeline;
-import com.google.common.collect.Sets;
-import java.util.Set;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-public enum TopicMessageType {
- UNSPECIFIED("UNSPECIFIED"),
- NORMAL("NORMAL"),
- FIFO("FIFO"),
- DELAY("DELAY"),
- TRANSACTION("TRANSACTION");
+public interface RequestPipeline {
- private final String value;
- TopicMessageType(String value) {
- this.value = value;
- }
-
- public static Set<String> topicMessageTypeSet() {
- return Sets.newHashSet(UNSPECIFIED.value, NORMAL.value, FIFO.value, DELAY.value, TRANSACTION.value);
- }
-
- public String getValue() {
- return value;
- }
+ void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception;
- public String getMetricsValue() {
- return value.toLowerCase();
+ default RequestPipeline pipe(RequestPipeline source) {
+ return (ctx, request, context) -> {
+ source.execute(ctx, request, context);
+ execute(ctx, request, context);
+ };
}
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java
index ff7ef01a0..04ad5e269 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java
@@ -175,6 +175,11 @@ public class SimpleChannel extends AbstractChannel {
return promise;
}
+ @Override
+ public boolean isWritable() {
+ return true;
+ }
+
public void updateLastAccessTime() {
this.lastAccessTime = System.currentTimeMillis();
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
index f27f002d0..c2a5a6435 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
@@ -63,13 +63,13 @@ public class ClusterMessageService implements MessageService {
CompletableFuture<List<SendResult>> future;
if (msgList.size() == 1) {
future = this.mqClientAPIFactory.getClient().sendMessageAsync(
- messageQueue.getBrokerAddr(),
- messageQueue.getBrokerName(), msgList.get(0), requestHeader, timeoutMillis)
+ messageQueue.getBrokerAddr(),
+ messageQueue.getBrokerName(), msgList.get(0), requestHeader, timeoutMillis)
.thenApply(Lists::newArrayList);
} else {
future = this.mqClientAPIFactory.getClient().sendMessageAsync(
- messageQueue.getBrokerAddr(),
- messageQueue.getBrokerName(), msgList, requestHeader, timeoutMillis)
+ messageQueue.getBrokerAddr(),
+ messageQueue.getBrokerName(), msgList, requestHeader, timeoutMillis)
.thenApply(Lists::newArrayList);
}
return future;
@@ -86,7 +86,8 @@ public class ClusterMessageService implements MessageService {
}
@Override
- public CompletableFuture<Void> endTransactionOneway(ProxyContext ctx, String brokerName, EndTransactionRequestHeader requestHeader,
+ public CompletableFuture<Void> endTransactionOneway(ProxyContext ctx, String brokerName,
+ EndTransactionRequestHeader requestHeader,
long timeoutMillis) {
CompletableFuture<Void> future = new CompletableFuture<>();
try {
@@ -205,6 +206,32 @@ public class ClusterMessageService implements MessageService {
);
}
+ @Override
+ public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis) {
+ try {
+ String brokerAddress = topicRouteService.getBrokerAddr(brokerName);
+ return mqClientAPIFactory.getClient().invoke(brokerAddress, request, timeoutMillis);
+ } catch (Exception e) {
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+ future.completeExceptionally(e);
+ return future;
+ }
+ }
+
+ @Override
+ public CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis) {
+ try {
+ String brokerAddress = topicRouteService.getBrokerAddr(brokerName);
+ return mqClientAPIFactory.getClient().invokeOneway(brokerAddress, request, timeoutMillis);
+ } catch (Exception e) {
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ future.completeExceptionally(e);
+ return future;
+ }
+ }
+
protected String resolveBrokerAddrInReceiptHandle(ReceiptHandle handle) {
try {
return this.topicRouteService.getBrokerAddr(handle.getBrokerName());
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
index 491926d01..115c140ff 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
@@ -402,4 +402,16 @@ public class LocalMessageService implements MessageService {
GetMinOffsetRequestHeader requestHeader, long timeoutMillis) {
throw new NotImplementedException("getMinOffset is not implemented in LocalMessageService");
}
+
+ @Override
+ public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis) {
+ throw new NotImplementedException("request is not implemented in LocalMessageService");
+ }
+
+ @Override
+ public CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis) {
+ throw new NotImplementedException("requestOneway is not implemented in LocalMessageService");
+ }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
index 18673b505..15da17154 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
@@ -139,4 +139,10 @@ public interface MessageService {
GetMinOffsetRequestHeader requestHeader,
long timeoutMillis
);
+
+ CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis);
+
+ CompletableFuture<Void> requestOneway(ProxyContext ctx, String brokerName, RemotingCommand request,
+ long timeoutMillis);
}
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
new file mode 100644
index 000000000..b581d8a91
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class AbstractRemotingActivityTest extends InitConfigAndLoggerTest {
+ AbstractRemotingActivity remotingActivity;
+ @Mock
+ MessagingProcessor messagingProcessorMock;
+ @Spy
+ ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, "1", "2")) {
+ @Override
+ public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {
+ return null;
+ }
+ };
+
+ @Before
+ public void setup() {
+ remotingActivity = new AbstractRemotingActivity(null, messagingProcessorMock) {
+ @Override
+ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+ ProxyContext context) throws Exception {
+ return null;
+ }
+ };
+ }
+
+ @Test
+ public void request() throws Exception {
+ String brokerName = "broker";
+ String remark = "success";
+ when(messagingProcessorMock.request(any(), anyString(), any(), anyLong())).thenReturn(CompletableFuture.completedFuture(
+ RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, remark)
+ ));
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);
+ RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
+ assertThat(remotingCommand).isNull();
+ verify(ctx, times(1)).writeAndFlush(any());
+ }
+}
\ No newline at end of file
[rocketmq] 15/26: [ISSUE #5486] Add AuthenticationPipeline
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 29c4122d9e7a919fee701ec692e5aa277c8e8ca9
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Thu Nov 17 17:07:45 2022 +0800
[ISSUE #5486] Add AuthenticationPipeline
---
.../proxy/remoting/RemotingProtocolServer.java | 10 ++++-
.../remoting/pipeline/AuthenticationPipeline.java | 46 ++++++++++++++++++++++
2 files changed, 55 insertions(+), 1 deletion(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
index a173a79b6..adbc21169 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -19,12 +19,16 @@ package org.apache.rocketmq.proxy.remoting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import io.netty.channel.Channel;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.acl.AccessValidator;
+import org.apache.rocketmq.acl.plain.PlainAccessValidator;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.future.FutureTaskExt;
@@ -46,6 +50,7 @@ import org.apache.rocketmq.proxy.remoting.activity.PullMessageActivity;
import org.apache.rocketmq.proxy.remoting.activity.SendMessageActivity;
import org.apache.rocketmq.proxy.remoting.activity.TransactionActivity;
import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;
+import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline;
import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
import org.apache.rocketmq.remoting.ChannelEventListener;
import org.apache.rocketmq.remoting.RemotingServer;
@@ -252,9 +257,12 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
RequestPipeline pipeline = (ctx, request, context) -> {
};
+ List<AccessValidator> accessValidatorList = new ArrayList<>();
+ accessValidatorList.add(new PlainAccessValidator());
+
// add pipeline
// the last pipe add will execute at the first
- return pipeline;
+ return pipeline.pipe(new AuthenticationPipeline(accessValidatorList));
}
protected class ThreadPoolHeadSlowTimeMillsMonitor implements ThreadPoolStatusMonitor {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java
new file mode 100644
index 000000000..4bcc1479d
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.pipeline;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.util.List;
+import org.apache.rocketmq.acl.AccessResource;
+import org.apache.rocketmq.acl.AccessValidator;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class AuthenticationPipeline implements RequestPipeline {
+ private final List<AccessValidator> accessValidatorList;
+
+ public AuthenticationPipeline(List<AccessValidator> accessValidatorList) {
+ this.accessValidatorList = accessValidatorList;
+ }
+
+ @Override
+ public void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception {
+ ProxyConfig config = ConfigurationManager.getProxyConfig();
+ if (config.isEnableACL()) {
+ for (AccessValidator accessValidator : accessValidatorList) {
+ AccessResource accessResource = accessValidator.parse(request, context.getRemoteAddress());
+ accessValidator.validate(accessResource);
+ }
+ }
+ }
+}
[rocketmq] 14/26: [ISSUE #5485] add test cases for channel management
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 834287af0c0cf7923cf884da8491329ca71f3a78
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Thu Nov 17 14:20:47 2022 +0800
[ISSUE #5485] add test cases for channel management
---
.../grpc/v2/channel/GrpcClientChannelTest.java | 82 ++++++
.../proxy/processor/channel/RemoteChannelTest.java | 50 ++++
.../channel/RemotingChannelManagerTest.java | 162 +++++++++++
.../remoting/channel/RemotingChannelTest.java | 80 ++++++
.../service/admin/DefaultAdminServiceTest.java | 103 +++++++
.../service/sysmessage/HeartbeatSyncerTest.java | 319 +++++++++++++++++++++
6 files changed, 796 insertions(+)
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java
new file mode 100644
index 000000000..70e10bc2b
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.grpc.v2.channel;
+
+import apache.rocketmq.v2.Publishing;
+import apache.rocketmq.v2.Resource;
+import apache.rocketmq.v2.Settings;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
+import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
+import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
+import org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;
+import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class GrpcClientChannelTest extends InitConfigAndLoggerTest {
+
+ @Mock
+ private ProxyRelayService proxyRelayService;
+ @Mock
+ private GrpcClientSettingsManager grpcClientSettingsManager;
+ @Mock
+ private GrpcChannelManager grpcChannelManager;
+
+ private String clientId;
+ private GrpcClientChannel grpcClientChannel;
+
+ @Before
+ public void before() throws Throwable {
+ super.before();
+ this.clientId = RandomStringUtils.randomAlphabetic(10);
+ this.grpcClientChannel = new GrpcClientChannel(proxyRelayService, grpcClientSettingsManager, grpcChannelManager,
+ ProxyContext.create().setRemoteAddress("10.152.39.53:9768").setLocalAddress("11.193.0.1:1210"),
+ this.clientId);
+ }
+
+ @Test
+ public void testChannelExtendAttributeParse() {
+ Settings clientSettings = Settings.newBuilder()
+ .setPublishing(Publishing.newBuilder()
+ .addTopics(Resource.newBuilder()
+ .setName("topic")
+ .build())
+ .build())
+ .build();
+ when(grpcClientSettingsManager.getRawClientSettings(eq(clientId))).thenReturn(clientSettings);
+
+ RemoteChannel remoteChannel = this.grpcClientChannel.toRemoteChannel();
+ assertEquals(ChannelProtocolType.GRPC_V2, remoteChannel.getType());
+ assertEquals(clientSettings, GrpcClientChannel.parseChannelExtendAttribute(remoteChannel));
+ assertEquals(clientSettings, GrpcClientChannel.parseChannelExtendAttribute(this.grpcClientChannel));
+ assertNull(GrpcClientChannel.parseChannelExtendAttribute(mock(RemotingChannel.class)));
+ }
+}
\ No newline at end of file
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelTest.java
new file mode 100644
index 000000000..d504fdc5f
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannelTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.processor.channel;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class RemoteChannelTest {
+
+ @Test
+ public void testEncodeAndDecode() {
+ String remoteProxyIp = "11.193.0.1";
+ String remoteAddress = "10.152.39.53:9768";
+ String localAddress = "11.193.0.1:1210";
+ ChannelProtocolType type = ChannelProtocolType.REMOTING;
+ String extendAttribute = RandomStringUtils.randomAlphabetic(10);
+ RemoteChannel remoteChannel = new RemoteChannel(remoteProxyIp, remoteAddress, localAddress, type, extendAttribute);
+
+ String encodedData = remoteChannel.encode();
+ assertNotNull(encodedData);
+
+ RemoteChannel decodedChannel = RemoteChannel.decode(encodedData);
+ assertEquals(remoteProxyIp, decodedChannel.remoteProxyIp);
+ assertEquals(remoteAddress, decodedChannel.getRemoteAddress());
+ assertEquals(localAddress, decodedChannel.getLocalAddress());
+ assertEquals(type, decodedChannel.type);
+ assertEquals(extendAttribute, decodedChannel.extendAttribute);
+
+ assertNull(RemoteChannel.decode(""));
+ }
+}
\ No newline at end of file
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManagerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManagerTest.java
new file mode 100644
index 000000000..5a5b441e9
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManagerTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.channel;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelId;
+import java.util.HashSet;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
+import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RemotingChannelManagerTest {
+ @Mock
+ private RemotingProxyOutClient remotingProxyOutClient;
+ @Mock
+ private ProxyRelayService proxyRelayService;
+
+ private final String remoteAddress = "10.152.39.53:9768";
+ private final String localAddress = "11.193.0.1:1210";
+ private RemotingChannelManager remotingChannelManager;
+
+ @Before
+ public void before() {
+ this.remotingChannelManager = new RemotingChannelManager(this.remotingProxyOutClient, this.proxyRelayService);
+ }
+
+ @Test
+ public void testCreateChannel() {
+ String group = "group";
+ String clientId = RandomStringUtils.randomAlphabetic(10);
+
+ Channel producerChannel = createMockChannel();
+ RemotingChannel producerRemotingChannel = this.remotingChannelManager.createProducerChannel(producerChannel, group, clientId);
+ assertNotNull(producerRemotingChannel);
+ assertSame(producerRemotingChannel, this.remotingChannelManager.createProducerChannel(producerChannel, group, clientId));
+
+ Channel consumerChannel = createMockChannel();
+ RemotingChannel consumerRemotingChannel = this.remotingChannelManager.createConsumerChannel(consumerChannel, group, clientId, new HashSet<>());
+ assertSame(consumerRemotingChannel, this.remotingChannelManager.createConsumerChannel(consumerChannel, group, clientId, new HashSet<>()));
+ assertNotNull(consumerRemotingChannel);
+
+ assertNotSame(producerRemotingChannel, consumerRemotingChannel);
+ }
+
+ @Test
+ public void testRemoveProducerChannel() {
+ String group = "group";
+ String clientId = RandomStringUtils.randomAlphabetic(10);
+
+ {
+ Channel producerChannel = createMockChannel();
+ RemotingChannel producerRemotingChannel = this.remotingChannelManager.createProducerChannel(producerChannel, group, clientId);
+ assertSame(producerRemotingChannel, this.remotingChannelManager.removeProducerChannel(group, producerRemotingChannel));
+ assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());
+ }
+ {
+ Channel producerChannel = createMockChannel();
+ RemotingChannel producerRemotingChannel = this.remotingChannelManager.createProducerChannel(producerChannel, group, clientId);
+ assertSame(producerRemotingChannel, this.remotingChannelManager.removeProducerChannel(group, producerChannel));
+ assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());
+ }
+ }
+
+ @Test
+ public void testRemoveConsumerChannel() {
+ String group = "group";
+ String clientId = RandomStringUtils.randomAlphabetic(10);
+
+ {
+ Channel consumerChannel = createMockChannel();
+ RemotingChannel consumerRemotingChannel = this.remotingChannelManager.createConsumerChannel(consumerChannel, group, clientId, new HashSet<>());
+ assertSame(consumerRemotingChannel, this.remotingChannelManager.removeConsumerChannel(group, consumerRemotingChannel));
+ assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());
+ }
+ {
+ Channel consumerChannel = createMockChannel();
+ RemotingChannel consumerRemotingChannel = this.remotingChannelManager.createConsumerChannel(consumerChannel, group, clientId, new HashSet<>());
+ assertSame(consumerRemotingChannel, this.remotingChannelManager.removeConsumerChannel(group, consumerChannel));
+ assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());
+ }
+ }
+
+ @Test
+ public void testRemoveChannel() {
+ String consumerGroup = "consumerGroup";
+ String producerGroup = "producerGroup";
+ String clientId = RandomStringUtils.randomAlphabetic(10);
+
+ Channel consumerChannel = createMockChannel();
+ RemotingChannel consumerRemotingChannel = this.remotingChannelManager.createConsumerChannel(consumerChannel, consumerGroup, clientId, new HashSet<>());
+ Channel producerChannel = createMockChannel();
+ RemotingChannel producerRemotingChannel = this.remotingChannelManager.createProducerChannel(producerChannel, producerGroup, clientId);
+
+ assertSame(consumerRemotingChannel, this.remotingChannelManager.removeChannel(consumerChannel).stream().findFirst().get());
+ assertSame(producerRemotingChannel, this.remotingChannelManager.removeChannel(producerChannel).stream().findFirst().get());
+
+ assertTrue(this.remotingChannelManager.groupChannelMap.isEmpty());
+ }
+
+ private Channel createMockChannel() {
+ return new MockChannel(RandomStringUtils.randomAlphabetic(10));
+ }
+
+ private class MockChannel extends SimpleChannel {
+
+ public MockChannel(String channelId) {
+ super(null, new MockChannelId(channelId), RemotingChannelManagerTest.this.remoteAddress, RemotingChannelManagerTest.this.localAddress);
+ }
+ }
+
+ private static class MockChannelId implements ChannelId {
+
+ private final String channelId;
+
+ public MockChannelId(String channelId) {
+ this.channelId = channelId;
+ }
+
+ @Override
+ public String asShortText() {
+ return channelId;
+ }
+
+ @Override
+ public String asLongText() {
+ return channelId;
+ }
+
+ @Override
+ public int compareTo(@NotNull ChannelId o) {
+ return this.channelId.compareTo(o.asLongText());
+ }
+ }
+}
\ No newline at end of file
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java
new file mode 100644
index 000000000..840f3e40f
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.channel;
+
+import io.netty.channel.Channel;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.rocketmq.common.filter.FilterAPI;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;
+import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
+import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
+import org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;
+import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
+import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RemotingChannelTest extends InitConfigAndLoggerTest {
+ @Mock
+ private RemotingProxyOutClient remotingProxyOutClient;
+ @Mock
+ private ProxyRelayService proxyRelayService;
+ @Mock
+ private Channel parent;
+
+ private String clientId;
+ private Set<SubscriptionData> subscriptionData;
+ private RemotingChannel remotingChannel;
+
+ private final String remoteAddress = "10.152.39.53:9768";
+ private final String localAddress = "11.193.0.1:1210";
+
+ @Before
+ public void before() throws Throwable {
+ super.before();
+ this.clientId = RandomStringUtils.randomAlphabetic(10);
+ when(parent.remoteAddress()).thenReturn(RemotingUtil.string2SocketAddress(remoteAddress));
+ when(parent.localAddress()).thenReturn(RemotingUtil.string2SocketAddress(localAddress));
+ this.subscriptionData = new HashSet<>();
+ this.subscriptionData.add(FilterAPI.buildSubscriptionData("topic", "subTag"));
+ this.remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService,
+ parent, clientId, subscriptionData);
+ }
+
+ @Test
+ public void testChannelExtendAttributeParse() {
+ RemoteChannel remoteChannel = this.remotingChannel.toRemoteChannel();
+ assertEquals(ChannelProtocolType.REMOTING, remoteChannel.getType());
+ assertEquals(subscriptionData, RemotingChannel.parseChannelExtendAttribute(remoteChannel));
+ assertEquals(subscriptionData, RemotingChannel.parseChannelExtendAttribute(this.remotingChannel));
+ assertNull(RemotingChannel.parseChannelExtendAttribute(mock(GrpcClientChannel.class)));
+ }
+}
\ No newline at end of file
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java
new file mode 100644
index 000000000..039efd8b4
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.admin;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIExt;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DefaultAdminServiceTest {
+ @Mock
+ private MQClientAPIFactory mqClientAPIFactory;
+ @Mock
+ private MQClientAPIExt mqClientAPIExt;
+
+ private DefaultAdminService defaultAdminService;
+
+ @Before
+ public void before() {
+ when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt);
+ defaultAdminService = new DefaultAdminService(mqClientAPIFactory);
+ }
+
+ @Test
+ public void testCreateTopic() throws Exception {
+ when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq("createTopic"), anyLong()))
+ .thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, ""))
+ .thenReturn(createTopicRouteData(1));
+ when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq("sampleTopic"), anyLong()))
+ .thenReturn(createTopicRouteData(2));
+
+ ArgumentCaptor<String> addrArgumentCaptor = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<TopicConfig> topicConfigArgumentCaptor = ArgumentCaptor.forClass(TopicConfig.class);
+ doNothing().when(mqClientAPIExt).createTopic(addrArgumentCaptor.capture(), anyString(), topicConfigArgumentCaptor.capture(), anyLong());
+
+ assertTrue(defaultAdminService.createTopicOnTopicBrokerIfNotExist(
+ "createTopic",
+ "sampleTopic",
+ 7,
+ 8,
+ true,
+ 1
+ ));
+
+ assertEquals(2, addrArgumentCaptor.getAllValues().size());
+ Set<String> createAddr = new HashSet<>(addrArgumentCaptor.getAllValues());
+ assertTrue(createAddr.contains("127.0.0.1:10911"));
+ assertTrue(createAddr.contains("127.0.0.2:10911"));
+ assertEquals("createTopic", topicConfigArgumentCaptor.getValue().getTopicName());
+ assertEquals(7, topicConfigArgumentCaptor.getValue().getWriteQueueNums());
+ assertEquals(8, topicConfigArgumentCaptor.getValue().getReadQueueNums());
+ }
+
+ private TopicRouteData createTopicRouteData(int brokerNum) {
+ TopicRouteData topicRouteData = new TopicRouteData();
+ for (int i = 0; i < brokerNum; i++) {
+ BrokerData brokerData = new BrokerData();
+ HashMap<Long, String> addrMap = new HashMap<>();
+ addrMap.put(0L, "127.0.0." + (i + 1) + ":10911");
+ brokerData.setBrokerAddrs(addrMap);
+ brokerData.setBrokerName("broker-" + i);
+ brokerData.setCluster("cluster");
+ topicRouteData.getBrokerDatas().add(brokerData);
+ }
+ return topicRouteData;
+ }
+}
\ No newline at end of file
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
new file mode 100644
index 000000000..8ac74f533
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.service.sysmessage;
+
+import apache.rocketmq.v2.FilterExpression;
+import apache.rocketmq.v2.FilterType;
+import apache.rocketmq.v2.Resource;
+import apache.rocketmq.v2.Settings;
+import apache.rocketmq.v2.Subscription;
+import apache.rocketmq.v2.SubscriptionEntry;
+import com.google.common.collect.Sets;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelId;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.filter.FilterAPI;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.QueueData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;
+import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;
+import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
+import org.apache.rocketmq.proxy.remoting.RemotingProxyOutClient;
+import org.apache.rocketmq.proxy.remoting.channel.RemotingChannel;
+import org.apache.rocketmq.proxy.service.admin.AdminService;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIExt;
+import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
+import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
+import org.apache.rocketmq.proxy.service.route.MessageQueueView;
+import org.apache.rocketmq.proxy.service.route.TopicRouteService;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
+import org.assertj.core.util.Lists;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class HeartbeatSyncerTest extends InitConfigAndLoggerTest {
+ @Mock
+ private TopicRouteService topicRouteService;
+ @Mock
+ private AdminService adminService;
+ @Mock
+ private ConsumerManagerInterface consumerManager;
+ @Mock
+ private MQClientAPIFactory mqClientAPIFactory;
+ @Mock
+ private MQClientAPIExt mqClientAPIExt;
+ @Mock
+ private ProxyRelayService proxyRelayService;
+
+ private String clientId;
+ private final String remoteAddress = "10.152.39.53:9768";
+ private final String localAddress = "11.193.0.1:1210";
+ private final String clusterName = "cluster";
+ private final String brokerName = "broker-01";
+
+ @Before
+ public void before() throws Throwable {
+ super.before();
+ this.clientId = RandomStringUtils.randomAlphabetic(10);
+ when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt);
+
+ {
+ TopicRouteData topicRouteData = new TopicRouteData();
+ QueueData queueData = new QueueData();
+ queueData.setReadQueueNums(8);
+ queueData.setWriteQueueNums(8);
+ queueData.setPerm(6);
+ queueData.setBrokerName(brokerName);
+ topicRouteData.getQueueDatas().add(queueData);
+ BrokerData brokerData = new BrokerData();
+ brokerData.setCluster(clusterName);
+ brokerData.setBrokerName(brokerName);
+ HashMap<Long, String> brokerAddr = new HashMap<>();
+ brokerAddr.put(0L, "127.0.0.1:10911");
+ brokerData.setBrokerAddrs(brokerAddr);
+ topicRouteData.getBrokerDatas().add(brokerData);
+ MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData);
+ when(this.topicRouteService.getAllMessageQueueView(anyString())).thenReturn(messageQueueView);
+ }
+ }
+
+ @Test
+ public void testSyncGrpcV2Channel() throws Exception {
+ String consumerGroup = "consumerGroup";
+ GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class);
+ GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class);
+ GrpcClientChannel grpcClientChannel = new GrpcClientChannel(
+ proxyRelayService, grpcClientSettingsManager, grpcChannelManager,
+ ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress),
+ clientId);
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+ grpcClientChannel,
+ clientId,
+ LanguageCode.JAVA,
+ 5
+ );
+
+ ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);
+ SendResult sendResult = new SendResult();
+ sendResult.setSendStatus(SendStatus.SEND_OK);
+ doReturn(CompletableFuture.completedFuture(sendResult)).when(this.mqClientAPIExt)
+ .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());
+
+ Settings settings = Settings.newBuilder()
+ .setSubscription(Subscription.newBuilder()
+ .addSubscriptions(SubscriptionEntry.newBuilder()
+ .setTopic(Resource.newBuilder().setName("topic").build())
+ .setExpression(FilterExpression.newBuilder()
+ .setType(FilterType.TAG)
+ .setExpression("tag")
+ .build())
+ .build())
+ .build())
+ .build();
+ when(grpcClientSettingsManager.getRawClientSettings(eq(clientId))).thenReturn(settings);
+
+ HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory);
+ heartbeatSyncer.onConsumerRegister(
+ consumerGroup,
+ clientChannelInfo,
+ ConsumeType.CONSUME_PASSIVELY,
+ MessageModel.CLUSTERING,
+ ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,
+ Sets.newHashSet(FilterAPI.buildSubscriptionData("topic", "tag"))
+ );
+
+ await().atMost(Duration.ofSeconds(3)).until(() -> !messageArgumentCaptor.getAllValues().isEmpty());
+ heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);
+ verify(consumerManager, never()).registerConsumer(anyString(), any(), any(), any(), any(), any(), anyBoolean());
+
+ String localServeAddr = ConfigurationManager.getProxyConfig().getLocalServeAddr();
+ // change local serve addr, to simulate other proxy receive messages
+ ConfigurationManager.getProxyConfig().setLocalServeAddr(RandomStringUtils.randomAlphabetic(10));
+ ArgumentCaptor<ClientChannelInfo> syncChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
+ doReturn(true).when(consumerManager).registerConsumer(anyString(), syncChannelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());
+
+ heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);
+ heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);
+ assertEquals(2, syncChannelInfoArgumentCaptor.getAllValues().size());
+ List<ClientChannelInfo> channelInfoList = syncChannelInfoArgumentCaptor.getAllValues();
+ assertSame(channelInfoList.get(0).getChannel(), channelInfoList.get(1).getChannel());
+ assertEquals(settings, GrpcClientChannel.parseChannelExtendAttribute(channelInfoList.get(0).getChannel()));
+ assertEquals(settings, GrpcClientChannel.parseChannelExtendAttribute(channelInfoList.get(1).getChannel()));
+
+ // start test sync client unregister
+ // reset localServeAddr
+ ConfigurationManager.getProxyConfig().setLocalServeAddr(localServeAddr);
+ heartbeatSyncer.onConsumerUnRegister(consumerGroup, clientChannelInfo);
+ await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 2);
+
+ ArgumentCaptor<ClientChannelInfo> syncUnRegisterChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
+ doNothing().when(consumerManager).unregisterConsumer(anyString(), syncUnRegisterChannelInfoArgumentCaptor.capture(), anyBoolean());
+
+ // change local serve addr, to simulate other proxy receive messages
+ ConfigurationManager.getProxyConfig().setLocalServeAddr(RandomStringUtils.randomAlphabetic(10));
+ heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getAllValues().get(1))), null);
+ assertSame(channelInfoList.get(0).getChannel(), syncUnRegisterChannelInfoArgumentCaptor.getValue().getChannel());
+ }
+
+ @Test
+ public void testSyncRemotingChannel() throws Exception {
+ String consumerGroup = "consumerGroup";
+ Set<SubscriptionData> subscriptionDataSet = new HashSet<>();
+ subscriptionDataSet.add(FilterAPI.buildSubscriptionData("topic", "tagSub"));
+ RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class);
+ RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, createMockChannel(), clientId, subscriptionDataSet);
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+ remotingChannel,
+ clientId,
+ LanguageCode.JAVA,
+ 4
+ );
+
+ ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);
+ SendResult sendResult = new SendResult();
+ sendResult.setSendStatus(SendStatus.SEND_OK);
+ doReturn(CompletableFuture.completedFuture(sendResult)).when(this.mqClientAPIExt)
+ .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());
+
+ HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory);
+ heartbeatSyncer.onConsumerRegister(
+ consumerGroup,
+ clientChannelInfo,
+ ConsumeType.CONSUME_PASSIVELY,
+ MessageModel.CLUSTERING,
+ ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,
+ subscriptionDataSet
+ );
+
+ await().atMost(Duration.ofSeconds(3)).until(() -> !messageArgumentCaptor.getAllValues().isEmpty());
+ heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);
+ verify(consumerManager, never()).registerConsumer(anyString(), any(), any(), any(), any(), any(), anyBoolean());
+
+ String localServeAddr = ConfigurationManager.getProxyConfig().getLocalServeAddr();
+ // change local serve addr, to simulate other proxy receive messages
+ ConfigurationManager.getProxyConfig().setLocalServeAddr(RandomStringUtils.randomAlphabetic(10));
+ ArgumentCaptor<ClientChannelInfo> syncChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
+ doReturn(true).when(consumerManager).registerConsumer(anyString(), syncChannelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());
+
+ heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);
+ heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getValue())), null);
+ assertEquals(2, syncChannelInfoArgumentCaptor.getAllValues().size());
+ List<ClientChannelInfo> channelInfoList = syncChannelInfoArgumentCaptor.getAllValues();
+ assertSame(channelInfoList.get(0).getChannel(), channelInfoList.get(1).getChannel());
+ assertEquals(subscriptionDataSet, RemotingChannel.parseChannelExtendAttribute(channelInfoList.get(0).getChannel()));
+ assertEquals(subscriptionDataSet, RemotingChannel.parseChannelExtendAttribute(channelInfoList.get(1).getChannel()));
+
+ // start test sync client unregister
+ // reset localServeAddr
+ ConfigurationManager.getProxyConfig().setLocalServeAddr(localServeAddr);
+ heartbeatSyncer.onConsumerUnRegister(consumerGroup, clientChannelInfo);
+ await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 2);
+
+ ArgumentCaptor<ClientChannelInfo> syncUnRegisterChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
+ doNothing().when(consumerManager).unregisterConsumer(anyString(), syncUnRegisterChannelInfoArgumentCaptor.capture(), anyBoolean());
+
+ // change local serve addr, to simulate other proxy receive messages
+ ConfigurationManager.getProxyConfig().setLocalServeAddr(RandomStringUtils.randomAlphabetic(10));
+ heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getAllValues().get(1))), null);
+ assertSame(channelInfoList.get(0).getChannel(), syncUnRegisterChannelInfoArgumentCaptor.getValue().getChannel());
+ }
+
+ private MessageExt convertFromMessage(Message message) {
+ MessageExt messageExt = new MessageExt();
+ messageExt.setTopic(message.getTopic());
+ messageExt.setBody(message.getBody());
+ return messageExt;
+ }
+
+ private Channel createMockChannel() {
+ return new MockChannel(RandomStringUtils.randomAlphabetic(10));
+ }
+
+ private class MockChannel extends SimpleChannel {
+
+ public MockChannel(String channelId) {
+ super(null, new MockChannelId(channelId), HeartbeatSyncerTest.this.remoteAddress, HeartbeatSyncerTest.this.localAddress);
+ }
+ }
+
+ private static class MockChannelId implements ChannelId {
+
+ private final String channelId;
+
+ public MockChannelId(String channelId) {
+ this.channelId = channelId;
+ }
+
+ @Override
+ public String asShortText() {
+ return channelId;
+ }
+
+ @Override
+ public String asLongText() {
+ return channelId;
+ }
+
+ @Override
+ public int compareTo(@NotNull ChannelId o) {
+ return this.channelId.compareTo(o.asLongText());
+ }
+ }
+}
\ No newline at end of file
[rocketmq] 05/26: [ISSUE #5485] test cases for client connection management
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 6c8233043676f42205d24f59a9981ba5a9e54366
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Wed Nov 9 11:37:36 2022 +0800
[ISSUE #5485] test cases for client connection management
---
.../test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java
index 635e4b5f3..524945bd6 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java
@@ -71,7 +71,7 @@ public class BaseActivityTest extends InitConfigTest {
metadata.put(InterceptorConstants.LOCAL_ADDRESS, LOCAL_ADDR);
when(messagingProcessor.getProxyRelayService()).thenReturn(proxyRelayService);
when(messagingProcessor.getMetadataService()).thenReturn(metadataService);
- grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService());
+ grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService(), grpcClientSettingsManager);
}
protected ProxyContext createContext() {
[rocketmq] 12/26: [ISSUE #5485] polish channel management which is been synced from other proxy
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 6351e1957ff6ac39e0e17c899fd0f82ec4ab600f
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Fri Nov 11 17:53:00 2022 +0800
[ISSUE #5485] polish channel management which is been synced from other proxy
---
.../proxy/grpc/v2/DefaultGrpcMessingActivity.java | 1 +
.../proxy/grpc/v2/client/ClientActivity.java | 30 +++-----------
.../grpc/v2/common/GrpcClientSettingsManager.java | 48 +++++++++++++++++++++-
.../proxy/processor/ReceiptHandleProcessor.java | 1 +
.../proxy/processor/channel/RemoteChannel.java | 18 ++++++++
.../remoting/activity/ClientManagerActivity.java | 1 +
.../proxy/remoting/channel/RemotingChannel.java | 12 ++++++
.../proxy/service/sysmessage/HeartbeatSyncer.java | 16 ++++----
.../service/sysmessage/HeartbeatSyncerData.java | 30 ++++++++++----
9 files changed, 113 insertions(+), 44 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
index f30519d74..194b9204f 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
@@ -91,6 +91,7 @@ public class DefaultGrpcMessingActivity extends AbstractStartAndShutdown impleme
this.clientActivity = new ClientActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
this.appendStartAndShutdown(this.receiptHandleProcessor);
+ this.appendStartAndShutdown(this.grpcClientSettingsManager);
}
@Override
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
index 00cc862a4..de8fba4a6 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
@@ -40,7 +40,6 @@ import java.util.concurrent.CompletableFuture;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
-import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
import org.apache.rocketmq.broker.client.ProducerChangeListener;
import org.apache.rocketmq.broker.client.ProducerGroupEvent;
@@ -436,32 +435,12 @@ public class ClientActivity extends AbstractMessingActivity {
}
if (args[0] instanceof ClientChannelInfo) {
ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
- String clientId = clientChannelInfo.getClientId();
if (ChannelHelper.isRemote(clientChannelInfo.getChannel())) {
- grpcClientSettingsManager.computeIfPresent(clientId, orgSettings -> {
- if (grpcChannelManager.getChannel(clientId) == null) {
- // if there is no channel connect directly to this proxy
- return null;
- }
- return orgSettings;
- });
- } else {
- grpcChannelManager.removeChannel(clientId);
- grpcClientSettingsManager.computeIfPresent(clientId, orgSettings -> {
- ConsumerGroupInfo consumerGroupInfo = messagingProcessor.getConsumerGroupInfo(group);
- if (consumerGroupInfo == null) {
- return null;
- }
- List<Channel> allChannels = consumerGroupInfo.getAllChannel();
- if (allChannels == null || allChannels.isEmpty() || allChannels.size() == 1) {
- // if there is only one channel of this clientId or
- // there is no channel if this clientId
- // remove settings of this client
- return null;
- }
- return orgSettings;
- });
+ return;
}
+ GrpcClientChannel removedChannel = grpcChannelManager.removeChannel(clientChannelInfo.getClientId());
+ log.info("remove grpc channel when client unregister. group:{}, clientChannelInfo:{}, removed:{}",
+ group, clientChannelInfo, removedChannel != null);
}
}
@@ -475,6 +454,7 @@ public class ClientActivity extends AbstractMessingActivity {
if (ChannelHelper.isRemote(channel)) {
// save settings from channel sync from other proxy
Settings settings = GrpcClientChannel.parseChannelExtendAttribute(channel);
+ log.debug("save client settings sync from other proxy. group:{}, channelInfo:{}, settings:{}", group, clientChannelInfo, settings);
if (settings == null) {
return;
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java
index 21c3395ae..b5b82fbdc 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java
@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.grpc.v2.common;
import apache.rocketmq.v2.Address;
import apache.rocketmq.v2.AddressScheme;
+import apache.rocketmq.v2.ClientType;
import apache.rocketmq.v2.CustomizedBackoff;
import apache.rocketmq.v2.Endpoints;
import apache.rocketmq.v2.ExponentialBackoff;
@@ -29,10 +30,18 @@ import com.google.protobuf.util.Durations;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.common.StartAndShutdown;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.config.MetricCollectorMode;
import org.apache.rocketmq.proxy.config.ProxyConfig;
@@ -43,8 +52,8 @@ import org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicy;
import org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicyType;
import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
-public class GrpcClientSettingsManager {
-
+public class GrpcClientSettingsManager extends ServiceThread implements StartAndShutdown {
+ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
protected static final Map<String, Settings> CLIENT_SETTINGS_MAP = new ConcurrentHashMap<>();
private final MessagingProcessor messagingProcessor;
@@ -205,4 +214,39 @@ public class GrpcClientSettingsManager {
}
return mergeMetric(settings);
}
+
+ @Override
+ public String getServiceName() {
+ return "GrpcClientSettingsManagerCleaner";
+ }
+
+ @Override
+ public void run() {
+ while (!this.isStopped()) {
+ this.waitForRunning(TimeUnit.SECONDS.toMillis(5));
+ }
+ }
+
+ @Override
+ protected void onWaitEnd() {
+ Set<String> clientIdSet = CLIENT_SETTINGS_MAP.keySet();
+ for (String clientId : clientIdSet) {
+ try {
+ CLIENT_SETTINGS_MAP.computeIfPresent(clientId, (clientIdKey, settings) -> {
+ if (!settings.getClientType().equals(ClientType.PUSH_CONSUMER) && !settings.getClientType().equals(ClientType.SIMPLE_CONSUMER)) {
+ return settings;
+ }
+ String consumerGroup = GrpcConverter.getInstance().wrapResourceWithNamespace(settings.getSubscription().getGroup());
+ ConsumerGroupInfo consumerGroupInfo = this.messagingProcessor.getConsumerGroupInfo(consumerGroup);
+ if (consumerGroupInfo == null || consumerGroupInfo.findChannel(clientId) == null) {
+ log.info("remove unused grpc client settings. group:{}, settings:{}", consumerGroupInfo, settings);
+ return null;
+ }
+ return settings;
+ });
+ } catch (Throwable t) {
+ log.error("check expired grpc client settings failed. clientId:{}", clientId, t);
+ }
+ }
+ }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java
index 9d000bfe9..5e096bc6b 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java
@@ -111,6 +111,7 @@ public class ReceiptHandleProcessor extends AbstractStartAndShutdown {
return;
}
clearGroup(new ReceiptHandleGroupKey(clientChannelInfo.getChannel(), group));
+ log.info("clear handle of this client when client unregister. group:{}, clientChannelInfo:{}", group, clientChannelInfo);
}
}
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java
index 5d9c9afcc..fb9666afc 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/channel/RemoteChannel.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.proxy.processor.channel;
+import com.google.common.base.MoreObjects;
import io.netty.channel.Channel;
import io.netty.channel.ChannelId;
import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
@@ -57,6 +58,13 @@ public class RemoteChannel extends SimpleChannel implements ChannelExtendAttribu
public int compareTo(ChannelId o) {
return this.id.compareTo(o.asLongText());
}
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("id", id)
+ .toString();
+ }
}
@Override
@@ -95,4 +103,14 @@ public class RemoteChannel extends SimpleChannel implements ChannelExtendAttribu
public String getChannelExtendAttribute() {
return this.extendAttribute;
}
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("channelId", id())
+ .add("type", type)
+ .add("remoteProxyIp", remoteProxyIp)
+ .add("extendAttribute", extendAttribute)
+ .toString();
+ }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
index 62400e033..10f7fa324 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
@@ -156,6 +156,7 @@ public class ClientManagerActivity extends AbstractRemotingActivity {
if (args[0] instanceof ClientChannelInfo) {
ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
remotingChannelManager.removeConsumerChannel(group, clientChannelInfo.getChannel());
+ log.info("remove remoting channel when client unregister. clientChannelInfo:{}", clientChannelInfo);
}
}
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
index 2b2cfca79..8b4832cad 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.remoting.channel;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
+import com.google.common.base.MoreObjects;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFutureListener;
import java.time.Duration;
@@ -221,4 +222,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver
ChannelProtocolType.REMOTING,
this.getChannelExtendAttribute());
}
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("parent", parent())
+ .add("clientId", clientId)
+ .add("remoteAddress", remoteAddress)
+ .add("localAddress", localAddress)
+ .add("subscriptionData", subscriptionData)
+ .toString();
+ }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
index ce3403766..12504a2f0 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
@@ -119,11 +119,11 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
messageModel,
consumeFromWhere,
proxyConfig.getLocalServeAddr(),
- remoteChannel.encode(),
- remoteChannel.getChannelExtendAttribute()
+ remoteChannel.encode()
);
data.setSubscriptionDataSet(subList);
+ log.debug("sync register heart beat. topic:{}, data:{}", this.getBroadcastTopicName(), data);
this.sendSystemMessage(data);
} catch (Throwable t) {
log.error("heartbeat register broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}, messageModel:{}, consumeFromWhere:{}, subList:{}",
@@ -158,10 +158,10 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
null,
null,
proxyConfig.getLocalServeAddr(),
- remoteChannel.encode(),
- remoteChannel.getChannelExtendAttribute()
+ remoteChannel.encode()
);
+ log.debug("sync unregister heart beat. topic:{}, data:{}", this.getBroadcastTopicName(), data);
this.sendSystemMessage(data);
} catch (Throwable t) {
log.error("heartbeat unregister broadcast failed. group:{}, clientChannelInfo:{}, consumeType:{}",
@@ -187,16 +187,16 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
continue;
}
- RemoteChannel channel = RemoteChannel.decode(data.getChannelData());
- RemoteChannel finalChannel = channel;
- channel = remoteChannelMap.computeIfAbsent(channel.id().asLongText(), key -> finalChannel);
- channel.setExtendAttribute(data.getChannelExtendAttribute());
+ RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData());
+ RemoteChannel channel = remoteChannelMap.computeIfAbsent(decodedChannel.id().asLongText(), key -> decodedChannel);
+ channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute());
ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
channel,
data.getClientId(),
data.getLanguage(),
data.getVersion()
);
+ log.debug("start process remote channel. data:{}, clientChannelInfo:{}", data, clientChannelInfo);
if (data.getHeartbeatType().equals(HeartbeatType.REGISTER)) {
this.consumerManager.registerConsumer(
data.getGroup(),
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
index 20fee7aac..f3b96ac9a 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.proxy.service.sysmessage;
+import com.google.common.base.MoreObjects;
import java.util.Set;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
@@ -37,13 +38,15 @@ public class HeartbeatSyncerData {
private ConsumeFromWhere consumeFromWhere;
private String connectProxyIp;
private String channelData;
- private String channelExtendAttribute;
+
+ public HeartbeatSyncerData() {
+ }
public HeartbeatSyncerData(HeartbeatType heartbeatType, String clientId,
LanguageCode language, int version, String group,
ConsumeType consumeType, MessageModel messageModel,
ConsumeFromWhere consumeFromWhere, String connectProxyIp,
- String channelData, String channelExtendAttribute) {
+ String channelData) {
this.heartbeatType = heartbeatType;
this.clientId = clientId;
this.language = language;
@@ -54,7 +57,6 @@ public class HeartbeatSyncerData {
this.consumeFromWhere = consumeFromWhere;
this.connectProxyIp = connectProxyIp;
this.channelData = channelData;
- this.channelExtendAttribute = channelExtendAttribute;
}
public HeartbeatType getHeartbeatType() {
@@ -154,11 +156,21 @@ public class HeartbeatSyncerData {
this.channelData = channelData;
}
- public String getChannelExtendAttribute() {
- return channelExtendAttribute;
- }
-
- public void setChannelExtendAttribute(String channelExtendAttribute) {
- this.channelExtendAttribute = channelExtendAttribute;
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("heartbeatType", heartbeatType)
+ .add("clientId", clientId)
+ .add("language", language)
+ .add("version", version)
+ .add("lastUpdateTimestamp", lastUpdateTimestamp)
+ .add("subscriptionDataSet", subscriptionDataSet)
+ .add("group", group)
+ .add("consumeType", consumeType)
+ .add("messageModel", messageModel)
+ .add("consumeFromWhere", consumeFromWhere)
+ .add("connectProxyIp", connectProxyIp)
+ .add("channelData", channelData)
+ .toString();
}
}
[rocketmq] 02/26: [ISSUE #5406] Support getConsumerIdList
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 4b9816e4a911f656470c70b4528e3a0517aa2090
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Mon Oct 31 16:44:50 2022 +0800
[ISSUE #5406] Support getConsumerIdList
---
.../remoting/activity/ConsumerManagerActivity.java | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
index fb248a894..734b1dad1 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ConsumerManagerActivity.java
@@ -20,11 +20,17 @@ package org.apache.rocketmq.proxy.remoting.activity;
import io.netty.channel.ChannelHandlerContext;
import java.time.Duration;
import java.util.ArrayList;
+import java.util.List;
import java.util.Set;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader;
+import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody;
+import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseHeader;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
@@ -64,8 +70,15 @@ public class ConsumerManagerActivity extends AbstractRemotingActivity {
protected RemotingCommand getConsumerListByGroup(ChannelHandlerContext ctx, RemotingCommand request,
ProxyContext context) throws Exception {
- // TODO after connection-related module
- return null;
+ RemotingCommand response = RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class);
+ GetConsumerListByGroupRequestHeader header = (GetConsumerListByGroupRequestHeader) request.decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class);
+ ConsumerGroupInfo consumerGroupInfo = messagingProcessor.getConsumerGroupInfo(header.getConsumerGroup());
+ List<String> clientIds = consumerGroupInfo.getAllClientId();
+ GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody();
+ body.setConsumerIdList(clientIds);
+ response.setBody(body.encode());
+ response.setCode(ResponseCode.SUCCESS);
+ return response;
}
protected RemotingCommand lockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request,
[rocketmq] 26/26: [ISSUE #5485] Fix ClusterGrpcIT
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit bdf5c95ce686ff2721d67b0ef040262b57691132
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Fri Dec 9 17:08:32 2022 +0800
[ISSUE #5485] Fix ClusterGrpcIT
---
.../rocketmq/proxy/service/ClusterServiceManager.java | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
index 24b27aaa2..bc3c58ed0 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
@@ -70,8 +70,6 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
public ClusterServiceManager(RPCHook rpcHook) {
ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
this.scheduledExecutorService = Executors.newScheduledThreadPool(3);
- this.producerManager = new ProducerManager();
- this.consumerManager = new ClusterConsumerManager(this.topicRouteService, this.adminService, this.operationClientAPIFactory, new ConsumerIdsChangeListenerImpl(), proxyConfig.getChannelExpiredTimeout());
this.messagingClientAPIFactory = new MQClientAPIFactory(
"ClusterMQClient_",
@@ -86,20 +84,24 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
rpcHook,
this.scheduledExecutorService
);
+
+ this.topicRouteService = new ClusterTopicRouteService(operationClientAPIFactory);
+ this.messageService = new ClusterMessageService(this.topicRouteService, this.messagingClientAPIFactory);
+ this.metadataService = new ClusterMetadataService(topicRouteService, operationClientAPIFactory);
+ this.adminService = new DefaultAdminService(this.operationClientAPIFactory);
+
+ this.producerManager = new ProducerManager();
+ this.consumerManager = new ClusterConsumerManager(this.topicRouteService, this.adminService, this.operationClientAPIFactory, new ConsumerIdsChangeListenerImpl(), proxyConfig.getChannelExpiredTimeout());
+
this.transactionClientAPIFactory = new MQClientAPIFactory(
"ClusterTransaction_",
1,
new ProxyClientRemotingProcessor(producerManager),
rpcHook,
scheduledExecutorService);
-
- this.topicRouteService = new ClusterTopicRouteService(operationClientAPIFactory);
- this.messageService = new ClusterMessageService(this.topicRouteService, this.messagingClientAPIFactory);
this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager, rpcHook,
this.transactionClientAPIFactory);
this.proxyRelayService = new ClusterProxyRelayService(this.clusterTransactionService);
- this.metadataService = new ClusterMetadataService(topicRouteService, operationClientAPIFactory);
- this.adminService = new DefaultAdminService(this.operationClientAPIFactory);
this.init();
}
[rocketmq] 20/26: [ISSUE #5406] Fix PullMessageActivityTest
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 1e71cd118e1bcc6dfd0f2034f391ee9fee862f4a
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Thu Nov 24 18:03:37 2022 +0800
[ISSUE #5406] Fix PullMessageActivityTest
---
.../rocketmq/proxy/remoting/activity/PullMessageActivityTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
index 5798e883b..d8ad45187 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
@@ -106,7 +106,7 @@ public class PullMessageActivityTest extends InitConfigTest {
newHeader.setQueueId(0);
newHeader.setQueueOffset(0L);
newHeader.setMaxMsgNums(16);
- newHeader.setSysFlag(PullSysFlag.buildSysFlag(true, false, false, false));
+ newHeader.setSysFlag(PullSysFlag.buildSysFlag(true, false, true, false));
newHeader.setCommitOffset(0L);
newHeader.setSuspendTimeoutMillis(1000L);
newHeader.setSubVersion(0L);
[rocketmq] 22/26: [ISSUE #5485] Fix by code review
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit b713aebfa71cc29ca195aca4664625f4c51ebe9c
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Tue Dec 6 14:24:34 2022 +0800
[ISSUE #5485] Fix by code review
---
.../apache/rocketmq/common/attribute/TopicMessageType.java | 3 +--
.../proxy/service/message/ClusterMessageService.java | 13 +++++--------
2 files changed, 6 insertions(+), 10 deletions(-)
diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java b/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
index 5e6629e3b..77629e4c9 100644
--- a/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
+++ b/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
@@ -53,9 +53,8 @@ public enum TopicMessageType {
return TopicMessageType.DELAY;
} else if (messageProperty.get(MessageConst.PROPERTY_SHARDING_KEY) != null) {
return TopicMessageType.FIFO;
- } else {
- return TopicMessageType.NORMAL;
}
+ return TopicMessageType.NORMAL;
}
public String getMetricsValue() {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
index c2a5a6435..872b16f51 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
@@ -30,6 +30,7 @@ import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.common.ProxyException;
import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
+import org.apache.rocketmq.proxy.common.utils.FutureUtils;
import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
import org.apache.rocketmq.proxy.service.route.TopicRouteService;
@@ -212,10 +213,8 @@ public class ClusterMessageService implements MessageService {
try {
String brokerAddress = topicRouteService.getBrokerAddr(brokerName);
return mqClientAPIFactory.getClient().invoke(brokerAddress, request, timeoutMillis);
- } catch (Exception e) {
- CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
- future.completeExceptionally(e);
- return future;
+ } catch (Throwable t) {
+ return FutureUtils.completeExceptionally(t);
}
}
@@ -225,10 +224,8 @@ public class ClusterMessageService implements MessageService {
try {
String brokerAddress = topicRouteService.getBrokerAddr(brokerName);
return mqClientAPIFactory.getClient().invokeOneway(brokerAddress, request, timeoutMillis);
- } catch (Exception e) {
- CompletableFuture<Void> future = new CompletableFuture<>();
- future.completeExceptionally(e);
- return future;
+ } catch (Throwable t) {
+ return FutureUtils.completeExceptionally(t);
}
}
[rocketmq] 18/26: [ISSUE #5406] Overwrite sysFlag to broker
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 9897e74a6c38200f6200c7bc69c7a118ec4ad7fd
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Fri Nov 18 15:46:44 2022 +0800
[ISSUE #5406] Overwrite sysFlag to broker
---
.../src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java | 4 ++++
.../apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java | 4 +++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java b/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java
index 20b8ad208..15d56dde7 100644
--- a/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java
+++ b/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java
@@ -77,6 +77,10 @@ public class PullSysFlag {
return (sysFlag & FLAG_SUBSCRIPTION) == FLAG_SUBSCRIPTION;
}
+ public static int buildSysFlagWithSubscription(final int sysFlag) {
+ return sysFlag | FLAG_SUBSCRIPTION;
+ }
+
public static boolean hasClassFilterFlag(final int sysFlag) {
return (sysFlag & FLAG_CLASS_FILTER) == FLAG_CLASS_FILTER;
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
index 873b52460..eb744676a 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
@@ -39,7 +39,8 @@ public class PullMessageActivity extends AbstractRemotingActivity {
protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
ProxyContext context) throws Exception {
PullMessageRequestHeader requestHeader = (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class);
- if (!PullSysFlag.hasSubscriptionFlag(requestHeader.getSysFlag())) {
+ int sysFlag = requestHeader.getSysFlag();
+ if (!PullSysFlag.hasSubscriptionFlag(sysFlag)) {
ConsumerGroupInfo consumerInfo = messagingProcessor.getConsumerGroupInfo(requestHeader.getConsumerGroup());
if (consumerInfo == null) {
return RemotingCommand.buildErrorResponse(ResponseCode.SUBSCRIPTION_NOT_LATEST,
@@ -50,6 +51,7 @@ public class PullMessageActivity extends AbstractRemotingActivity {
return RemotingCommand.buildErrorResponse(ResponseCode.SUBSCRIPTION_NOT_EXIST,
"the consumer's subscription not exist");
}
+ requestHeader.setSysFlag(PullSysFlag.buildSysFlagWithSubscription(sysFlag));
requestHeader.setSubscription(subscriptionData.getSubString());
requestHeader.setExpressionType(subscriptionData.getExpressionType());
request.writeCustomHeader(requestHeader);
[rocketmq] 11/26: [ISSUE #5406] use remotingAccessAddr and ListenPort in topicRoute for remoting proxy
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 21a01e3c628c34438e36dc3a715d044d41ff875f
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Wed Nov 9 18:04:17 2022 +0800
[ISSUE #5406] use remotingAccessAddr and ListenPort in topicRoute for remoting proxy
---
.../apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
index 26d28bafe..670d9735c 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
@@ -50,7 +50,7 @@ public class GetTopicRouteActivity extends AbstractRemotingActivity {
(GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);
List<Address> addressList = new ArrayList<>();
// AddressScheme is just a placeholder and will not affect topic route result in this case.
- addressList.add(new Address(Address.AddressScheme.IPv4, HostAndPort.fromString(proxyConfig.getRemotingAccessPoint())));
+ addressList.add(new Address(Address.AddressScheme.IPv4, HostAndPort.fromParts(proxyConfig.getRemotingAccessAddr(), proxyConfig.getRemotingListenPort())));
ProxyTopicRouteData proxyTopicRouteData = messagingProcessor.getTopicRouteDataForProxy(context, addressList, requestHeader.getTopic());
TopicRouteData topicRouteData = proxyTopicRouteData.buildTopicRouteData();
[rocketmq] 03/26: [ISSUE #5406] Add unit test
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit b887313bfbcb2d90c6db9757299fd76deb9f103d
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Tue Nov 1 15:14:23 2022 +0800
[ISSUE #5406] Add unit test
---
.../activity/AbstractRemotingActivity.java | 1 +
.../remoting/activity/GetTopicRouteActivity.java | 1 +
.../remoting/activity/PullMessageActivity.java | 15 +-
.../activity/AbstractRemotingActivityTest.java | 130 +++++++++++++++-
.../remoting/activity/PullMessageActivityTest.java | 165 +++++++++++++++++++++
.../remoting/activity/SendMessageActivityTest.java | 102 +++++++++++++
6 files changed, 394 insertions(+), 20 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
index 7f0d891ec..54ef7bfa7 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
@@ -67,6 +67,7 @@ public abstract class AbstractRemotingActivity implements NettyRequestProcessor
}
String brokerName = request.getExtFields().get(BROKER_NAME_FIELD);
if (request.isOnewayRPC()) {
+ messagingProcessor.requestOneway(context, brokerName, request, timeoutMillis);
return null;
}
messagingProcessor.request(context, brokerName, request, timeoutMillis)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
index d3b7de98d..26d28bafe 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
@@ -49,6 +49,7 @@ public class GetTopicRouteActivity extends AbstractRemotingActivity {
final GetRouteInfoRequestHeader requestHeader =
(GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);
List<Address> addressList = new ArrayList<>();
+ // AddressScheme is just a placeholder and will not affect topic route result in this case.
addressList.add(new Address(Address.AddressScheme.IPv4, HostAndPort.fromString(proxyConfig.getRemotingAccessPoint())));
ProxyTopicRouteData proxyTopicRouteData = messagingProcessor.getTopicRouteDataForProxy(context, addressList, requestHeader.getTopic());
TopicRouteData topicRouteData = proxyTopicRouteData.buildTopicRouteData();
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
index 819bf139d..873b52460 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivity.java
@@ -19,7 +19,6 @@ package org.apache.rocketmq.proxy.remoting.activity;
import io.netty.channel.ChannelHandlerContext;
import java.time.Duration;
-import java.util.concurrent.CompletableFuture;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
@@ -39,10 +38,6 @@ public class PullMessageActivity extends AbstractRemotingActivity {
@Override
protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
ProxyContext context) throws Exception {
- if (request.getExtFields().get(BROKER_NAME_FIELD) == null) {
- return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,
- "Request doesn't have field bname");
- }
PullMessageRequestHeader requestHeader = (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class);
if (!PullSysFlag.hasSubscriptionFlag(requestHeader.getSysFlag())) {
ConsumerGroupInfo consumerInfo = messagingProcessor.getConsumerGroupInfo(requestHeader.getConsumerGroup());
@@ -57,16 +52,10 @@ public class PullMessageActivity extends AbstractRemotingActivity {
}
requestHeader.setSubscription(subscriptionData.getSubString());
requestHeader.setExpressionType(subscriptionData.getExpressionType());
+ request.writeCustomHeader(requestHeader);
request.makeCustomHeaderToNet();
}
- String brokerName = requestHeader.getBname();
long timeoutMillis = requestHeader.getSuspendTimeoutMillis() + Duration.ofSeconds(10).toMillis();
- CompletableFuture<RemotingCommand> future = messagingProcessor.request(context, brokerName, request, timeoutMillis);
- future.thenAccept(r -> writeResponse(ctx, context, request, r))
- .exceptionally(t -> {
- writeErrResponse(ctx, context, request, t);
- return null;
- });
- return null;
+ return request(ctx, request, context, timeoutMillis);
}
}
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
index b581d8a91..74eb3cbd8 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
@@ -21,17 +21,24 @@ import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPromise;
import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.common.ProxyException;
+import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
import org.apache.rocketmq.proxy.processor.MessagingProcessor;
import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
@@ -39,7 +46,8 @@ import org.mockito.junit.MockitoJUnitRunner;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -50,7 +58,7 @@ public class AbstractRemotingActivityTest extends InitConfigAndLoggerTest {
@Mock
MessagingProcessor messagingProcessorMock;
@Spy
- ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, "1", "2")) {
+ ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, "0.0.0.0:0", "1.1.1.1:1")) {
@Override
public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {
return null;
@@ -69,16 +77,124 @@ public class AbstractRemotingActivityTest extends InitConfigAndLoggerTest {
}
@Test
- public void request() throws Exception {
+ public void testCreateContext() throws Exception {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ ProxyContext context = remotingActivity.createContext(ctx, request);
+ assertThat(context.getLanguage()).isEqualTo(LanguageCode.JAVA.name());
+ assertThat(context.getAction()).isEqualTo("Remoting" + RequestCode.PULL_MESSAGE);
+ }
+
+ @Test
+ public void testRequest() throws Exception {
String brokerName = "broker";
- String remark = "success";
- when(messagingProcessorMock.request(any(), anyString(), any(), anyLong())).thenReturn(CompletableFuture.completedFuture(
- RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, remark)
+ RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "remark");
+ when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(CompletableFuture.completedFuture(
+ response
));
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);
RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
assertThat(remotingCommand).isNull();
- verify(ctx, times(1)).writeAndFlush(any());
+ verify(ctx, times(1)).writeAndFlush(response);
+ }
+
+ @Test
+ public void testRequestOneway() throws Exception {
+ String brokerName = "broker";
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ request.markOnewayRPC();
+ request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);
+ RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
+ assertThat(remotingCommand).isNull();
+ verify(messagingProcessorMock, times(1)).requestOneway(any(), eq(brokerName), any(), anyLong());
+ }
+
+ @Test
+ public void testRequestInvalid() throws Exception {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ request.addExtField("test", "test");
+ RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
+ assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.VERSION_NOT_SUPPORTED);
+ verify(ctx, never()).writeAndFlush(any());
+ }
+
+ @Test
+ public void testRequestProxyException() throws Exception {
+ ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);
+ String brokerName = "broker";
+ String remark = "exception";
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+ future.completeExceptionally(new ProxyException(ProxyExceptionCode.FORBIDDEN, remark));
+ when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);
+ RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
+ assertThat(remotingCommand).isNull();
+ verify(ctx, times(1)).writeAndFlush(captor.capture());
+ assertThat(captor.getValue().getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
+ }
+
+ @Test
+ public void testRequestClientException() throws Exception {
+ ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);
+ String brokerName = "broker";
+ String remark = "exception";
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+ future.completeExceptionally(new MQClientException(remark, null));
+ when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);
+ RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
+ assertThat(remotingCommand).isNull();
+ verify(ctx, times(1)).writeAndFlush(captor.capture());
+ assertThat(captor.getValue().getCode()).isEqualTo(-1);
+ }
+
+ @Test
+ public void testRequestBrokerException() throws Exception {
+ ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);
+ String brokerName = "broker";
+ String remark = "exception";
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+ future.completeExceptionally(new MQBrokerException(ResponseCode.FLUSH_DISK_TIMEOUT, remark));
+ when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);
+ RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
+ assertThat(remotingCommand).isNull();
+ verify(ctx, times(1)).writeAndFlush(captor.capture());
+ assertThat(captor.getValue().getCode()).isEqualTo(ResponseCode.FLUSH_DISK_TIMEOUT);
+ }
+
+ @Test
+ public void testRequestAclException() throws Exception {
+ ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);
+ String brokerName = "broker";
+ String remark = "exception";
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+ future.completeExceptionally(new AclException(remark, ResponseCode.MESSAGE_ILLEGAL));
+ when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);
+ RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
+ assertThat(remotingCommand).isNull();
+ verify(ctx, times(1)).writeAndFlush(captor.capture());
+ assertThat(captor.getValue().getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
+ }
+
+ @Test
+ public void testRequestDefaultException() throws Exception {
+ ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);
+ String brokerName = "broker";
+ String remark = "exception";
+ CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+ future.completeExceptionally(new Exception(remark));
+ when(messagingProcessorMock.request(any(), eq(brokerName), any(), anyLong())).thenReturn(future);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+ request.addExtField(AbstractRemotingActivity.BROKER_NAME_FIELD, brokerName);
+ RemotingCommand remotingCommand = remotingActivity.request(ctx, request, null, 10000);
+ assertThat(remotingCommand).isNull();
+ verify(ctx, times(1)).writeAndFlush(captor.capture());
+ assertThat(captor.getValue().getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
}
}
\ No newline at end of file
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
new file mode 100644
index 000000000..ffbe2ffac
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/PullMessageActivityTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.common.sysflag.PullSysFlag;
+import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PullMessageActivityTest extends InitConfigAndLoggerTest {
+ PullMessageActivity pullMessageActivity;
+
+ @Mock
+ MessagingProcessor messagingProcessorMock;
+ @Mock
+ ConsumerGroupInfo consumerGroupInfoMock;
+
+ String topic = "topic";
+ String group = "group";
+ String brokerName = "brokerName";
+ String subString = "sub";
+ String type = "type";
+ @Spy
+ ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, "1", "2")) {
+ @Override
+ public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {
+ return null;
+ }
+ };
+
+ @Before
+ public void setup() throws Exception {
+ pullMessageActivity = new PullMessageActivity(null, messagingProcessorMock);
+ }
+
+ @Test
+ public void testPullMessageWithoutSub() throws Exception {
+ when(messagingProcessorMock.getConsumerGroupInfo(eq(group)))
+ .thenReturn(consumerGroupInfoMock);
+ SubscriptionData subscriptionData = new SubscriptionData();
+ subscriptionData.setSubString(subString);
+ subscriptionData.setExpressionType(type);
+ when(consumerGroupInfoMock.findSubscriptionData(eq(topic)))
+ .thenReturn(subscriptionData);
+
+ PullMessageRequestHeader header = new PullMessageRequestHeader();
+ header.setTopic(topic);
+ header.setConsumerGroup(group);
+ header.setQueueId(0);
+ header.setQueueOffset(0L);
+ header.setMaxMsgNums(16);
+ header.setSysFlag(PullSysFlag.buildSysFlag(true, false, false, false));
+ header.setCommitOffset(0L);
+ header.setSuspendTimeoutMillis(1000L);
+ header.setSubVersion(0L);
+ header.setBname(brokerName);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, header);
+ request.makeCustomHeaderToNet();
+ RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.NO_MESSAGE, "success");
+ PullMessageRequestHeader newHeader = new PullMessageRequestHeader();
+ newHeader.setTopic(topic);
+ newHeader.setConsumerGroup(group);
+ newHeader.setQueueId(0);
+ newHeader.setQueueOffset(0L);
+ newHeader.setMaxMsgNums(16);
+ newHeader.setSysFlag(PullSysFlag.buildSysFlag(true, false, false, false));
+ newHeader.setCommitOffset(0L);
+ newHeader.setSuspendTimeoutMillis(1000L);
+ newHeader.setSubVersion(0L);
+ newHeader.setBname(brokerName);
+ newHeader.setSubscription(subString);
+ newHeader.setExpressionType(type);
+ RemotingCommand matchRequest = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, newHeader);
+ matchRequest.setOpaque(request.getOpaque());
+ matchRequest.makeCustomHeaderToNet();
+
+ ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);
+ when(messagingProcessorMock.request(any(), eq(brokerName), captor.capture(), anyLong()))
+ .thenReturn(CompletableFuture.completedFuture(expectResponse));
+ RemotingCommand response = pullMessageActivity.processRequest0(ctx, request, null);
+ assertThat(captor.getValue().getExtFields()).isEqualTo(matchRequest.getExtFields());
+ assertThat(response).isNull();
+ verify(ctx, times(1)).writeAndFlush(eq(expectResponse));
+ }
+
+ @Test
+ public void testPullMessageWithSub() throws Exception {
+ when(messagingProcessorMock.getConsumerGroupInfo(eq(group)))
+ .thenReturn(consumerGroupInfoMock);
+ SubscriptionData subscriptionData = new SubscriptionData();
+ subscriptionData.setSubString(subString);
+ subscriptionData.setExpressionType(type);
+ when(consumerGroupInfoMock.findSubscriptionData(eq(topic)))
+ .thenReturn(subscriptionData);
+
+ PullMessageRequestHeader header = new PullMessageRequestHeader();
+ header.setTopic(topic);
+ header.setConsumerGroup(group);
+ header.setQueueId(0);
+ header.setQueueOffset(0L);
+ header.setMaxMsgNums(16);
+ header.setSysFlag(PullSysFlag.buildSysFlag(true, true, false, false));
+ header.setCommitOffset(0L);
+ header.setSuspendTimeoutMillis(1000L);
+ header.setSubVersion(0L);
+ header.setBname(brokerName);
+ header.setSubscription(subString);
+ header.setExpressionType(type);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, header);
+ request.makeCustomHeaderToNet();
+ RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.NO_MESSAGE, "success");
+
+ ArgumentCaptor<RemotingCommand> captor = ArgumentCaptor.forClass(RemotingCommand.class);
+ when(messagingProcessorMock.request(any(), eq(brokerName), captor.capture(), anyLong()))
+ .thenReturn(CompletableFuture.completedFuture(expectResponse));
+ RemotingCommand response = pullMessageActivity.processRequest0(ctx, request, null);
+ assertThat(captor.getValue().getExtFields()).isEqualTo(request.getExtFields());
+ assertThat(response).isNull();
+ verify(ctx, times(1)).writeAndFlush(eq(expectResponse));
+ }
+}
\ No newline at end of file
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java
new file mode 100644
index 000000000..e03bc26e0
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivityTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.common.attribute.TopicMessageType;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
+import org.apache.rocketmq.proxy.service.metadata.MetadataService;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SendMessageActivityTest extends InitConfigAndLoggerTest {
+ SendMessageActivity sendMessageActivity;
+
+ @Mock
+ MessagingProcessor messagingProcessorMock;
+ @Mock
+ MetadataService metadataServiceMock;
+
+ String topic = "topic";
+ String producerGroup = "group";
+ String brokerName = "brokerName";
+ @Spy
+ ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, "1", "2")) {
+ @Override
+ public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {
+ return null;
+ }
+ };
+
+ @Before
+ public void setup() {
+ sendMessageActivity = new SendMessageActivity(null, messagingProcessorMock);
+ when(messagingProcessorMock.getMetadataService()).thenReturn(metadataServiceMock);
+ }
+
+ @Test
+ public void testSendMessage() throws Exception {
+ when(metadataServiceMock.getTopicMessageType(eq(topic))).thenReturn(TopicMessageType.NORMAL);
+ Message message = new Message(topic, "123".getBytes());
+ message.putUserProperty("a", "b");
+ SendMessageRequestHeader sendMessageRequestHeader = new SendMessageRequestHeader();
+ sendMessageRequestHeader.setTopic(topic);
+ sendMessageRequestHeader.setProducerGroup(producerGroup);
+ sendMessageRequestHeader.setDefaultTopic("");
+ sendMessageRequestHeader.setDefaultTopicQueueNums(0);
+ sendMessageRequestHeader.setQueueId(0);
+ sendMessageRequestHeader.setSysFlag(0);
+ sendMessageRequestHeader.setBname(brokerName);
+ sendMessageRequestHeader.setProperties(MessageDecoder.messageProperties2String(message.getProperties()));
+ RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, sendMessageRequestHeader);
+ remotingCommand.setBody(message.getBody());
+ remotingCommand.makeCustomHeaderToNet();
+
+ RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "success");
+ when(messagingProcessorMock.request(any(), eq(brokerName), eq(remotingCommand), anyLong()))
+ .thenReturn(CompletableFuture.completedFuture(expectResponse));
+ RemotingCommand response = sendMessageActivity.processRequest0(ctx, remotingCommand, null);
+ assertThat(response).isNull();
+ verify(ctx, times(1)).writeAndFlush(eq(expectResponse));
+ }
+}
\ No newline at end of file
[rocketmq] 24/26: [ISSUE #5485] Use local address, remoting port and grpc port to build unique local proxy Id
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 4b311ff4dd36cb2ad6ef160efae9f07d6985b286
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Thu Dec 8 19:44:43 2022 +0800
[ISSUE #5485] Use local address, remoting port and grpc port to build unique local proxy Id
---
.../proxy/service/sysmessage/HeartbeatSyncer.java | 16 +++++++++++-----
.../proxy/service/sysmessage/HeartbeatSyncerData.java | 16 ++++++++--------
.../proxy/service/sysmessage/HeartbeatSyncerTest.java | 8 ++++----
3 files changed, 23 insertions(+), 17 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
index 9e333902e..51af02170 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
@@ -50,11 +50,13 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
protected ThreadPoolExecutor threadPoolExecutor;
protected ConsumerManager consumerManager;
protected final Map<String /* channelId as longText */, RemoteChannel> remoteChannelMap = new ConcurrentHashMap<>();
+ protected String localProxyId;
public HeartbeatSyncer(TopicRouteService topicRouteService, AdminService adminService,
ConsumerManager consumerManager, MQClientAPIFactory mqClientAPIFactory) {
super(topicRouteService, adminService, mqClientAPIFactory);
this.consumerManager = consumerManager;
+ this.localProxyId = buildLocalProxyId();
this.init();
}
@@ -104,7 +106,6 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
try {
this.threadPoolExecutor.submit(() -> {
try {
- ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
RemoteChannel remoteChannel = RemoteChannel.create(clientChannelInfo.getChannel());
if (remoteChannel == null) {
return;
@@ -118,7 +119,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
consumeType,
messageModel,
consumeFromWhere,
- proxyConfig.getLocalServeAddr(),
+ localProxyId,
remoteChannel.encode()
);
data.setSubscriptionDataSet(subList);
@@ -143,7 +144,6 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
try {
this.threadPoolExecutor.submit(() -> {
try {
- ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
RemoteChannel remoteChannel = RemoteChannel.create(clientChannelInfo.getChannel());
if (remoteChannel == null) {
return;
@@ -157,7 +157,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
null,
null,
null,
- proxyConfig.getLocalServeAddr(),
+ localProxyId,
remoteChannel.encode()
);
@@ -183,7 +183,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
for (MessageExt msg : msgs) {
try {
HeartbeatSyncerData data = JSON.parseObject(new String(msg.getBody(), StandardCharsets.UTF_8), HeartbeatSyncerData.class);
- if (data.getConnectProxyIp().equals(ConfigurationManager.getProxyConfig().getLocalServeAddr())) {
+ if (data.getLocalProxyId().equals(localProxyId)) {
continue;
}
@@ -221,4 +221,10 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
+
+ private String buildLocalProxyId() {
+ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
+ // use local address, remoting port and grpc port to build unique local proxy Id
+ return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort();
+ }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
index 10c6f1206..97760506f 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerData.java
@@ -36,7 +36,7 @@ public class HeartbeatSyncerData {
private ConsumeType consumeType;
private MessageModel messageModel;
private ConsumeFromWhere consumeFromWhere;
- private String connectProxyIp;
+ private String localProxyId;
private String channelData;
public HeartbeatSyncerData() {
@@ -45,7 +45,7 @@ public class HeartbeatSyncerData {
public HeartbeatSyncerData(HeartbeatType heartbeatType, String clientId,
LanguageCode language, int version, String group,
ConsumeType consumeType, MessageModel messageModel,
- ConsumeFromWhere consumeFromWhere, String connectProxyIp,
+ ConsumeFromWhere consumeFromWhere, String localProxyId,
String channelData) {
this.heartbeatType = heartbeatType;
this.clientId = clientId;
@@ -55,7 +55,7 @@ public class HeartbeatSyncerData {
this.consumeType = consumeType;
this.messageModel = messageModel;
this.consumeFromWhere = consumeFromWhere;
- this.connectProxyIp = connectProxyIp;
+ this.localProxyId = localProxyId;
this.channelData = channelData;
}
@@ -140,12 +140,12 @@ public class HeartbeatSyncerData {
this.consumeFromWhere = consumeFromWhere;
}
- public String getConnectProxyIp() {
- return connectProxyIp;
+ public String getLocalProxyId() {
+ return localProxyId;
}
- public void setConnectProxyIp(String connectProxyIp) {
- this.connectProxyIp = connectProxyIp;
+ public void setLocalProxyId(String localProxyId) {
+ this.localProxyId = localProxyId;
}
public String getChannelData() {
@@ -169,7 +169,7 @@ public class HeartbeatSyncerData {
.add("consumeType", consumeType)
.add("messageModel", messageModel)
.add("consumeFromWhere", consumeFromWhere)
- .add("connectProxyIp", connectProxyIp)
+ .add("connectProxyIp", localProxyId)
.add("channelData", channelData)
.toString();
}
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
index 95152186d..df98f31dc 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
@@ -185,7 +185,7 @@ public class HeartbeatSyncerTest extends InitConfigTest {
String localServeAddr = ConfigurationManager.getProxyConfig().getLocalServeAddr();
// change local serve addr, to simulate other proxy receive messages
- ConfigurationManager.getProxyConfig().setLocalServeAddr(RandomStringUtils.randomAlphabetic(10));
+ heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);
ArgumentCaptor<ClientChannelInfo> syncChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
doReturn(true).when(consumerManager).registerConsumer(anyString(), syncChannelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());
@@ -207,7 +207,7 @@ public class HeartbeatSyncerTest extends InitConfigTest {
doNothing().when(consumerManager).unregisterConsumer(anyString(), syncUnRegisterChannelInfoArgumentCaptor.capture(), anyBoolean());
// change local serve addr, to simulate other proxy receive messages
- ConfigurationManager.getProxyConfig().setLocalServeAddr(RandomStringUtils.randomAlphabetic(10));
+ heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);
heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getAllValues().get(1))), null);
assertSame(channelInfoList.get(0).getChannel(), syncUnRegisterChannelInfoArgumentCaptor.getValue().getChannel());
}
@@ -248,7 +248,7 @@ public class HeartbeatSyncerTest extends InitConfigTest {
String localServeAddr = ConfigurationManager.getProxyConfig().getLocalServeAddr();
// change local serve addr, to simulate other proxy receive messages
- ConfigurationManager.getProxyConfig().setLocalServeAddr(RandomStringUtils.randomAlphabetic(10));
+ heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);
ArgumentCaptor<ClientChannelInfo> syncChannelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
doReturn(true).when(consumerManager).registerConsumer(anyString(), syncChannelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());
@@ -270,7 +270,7 @@ public class HeartbeatSyncerTest extends InitConfigTest {
doNothing().when(consumerManager).unregisterConsumer(anyString(), syncUnRegisterChannelInfoArgumentCaptor.capture(), anyBoolean());
// change local serve addr, to simulate other proxy receive messages
- ConfigurationManager.getProxyConfig().setLocalServeAddr(RandomStringUtils.randomAlphabetic(10));
+ heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);
heartbeatSyncer.consumeMessage(Lists.newArrayList(convertFromMessage(messageArgumentCaptor.getAllValues().get(1))), null);
assertSame(channelInfoList.get(0).getChannel(), syncUnRegisterChannelInfoArgumentCaptor.getValue().getChannel());
}
[rocketmq] 21/26: [ISSUE #5485] Fix GrpcBaseIT
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 23c8efd801e60d91f1a2ab0a7d74a835c56efbeb
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Mon Nov 28 15:54:06 2022 +0800
[ISSUE #5485] Fix GrpcBaseIT
---
.../apache/rocketmq/proxy/config/ProxyConfig.java | 26 +++++++++++++++-------
.../sysmessage/AbstractSystemMessageSyncer.java | 16 ++++---------
.../apache/rocketmq/test/grpc/v2/GrpcBaseIT.java | 1 +
3 files changed, 23 insertions(+), 20 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
index e0f971202..4d5084cfd 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
@@ -54,16 +54,18 @@ public class ProxyConfig implements ConfigFile {
}
}
- private String rocketMQClusterName = "";
+ private String rocketMQClusterName = DEFAULT_CLUSTER_NAME;
private String proxyClusterName = DEFAULT_CLUSTER_NAME;
private String proxyName = StringUtils.isEmpty(localHostName) ? "DEFAULT_PROXY" : localHostName;
private String localServeAddr = "";
- private String systemTopicClusterName = "";
+ private String heartbeatSyncerTopicClusterName = "";
private int heartbeatSyncerThreadPoolNums = 4;
private int heartbeatSyncerThreadPoolQueueCapacity = 100;
+ private String heartbeatSyncerTopicName = "DefaultHeartBeatSyncerTopic";
+
/**
* configuration for ThreadPoolMonitor
*/
@@ -250,8 +252,8 @@ public class ProxyConfig implements ConfigFile {
if (StringUtils.isBlank(remotingAccessAddr)) {
this.remotingAccessAddr = this.localServeAddr;
}
- if (StringUtils.isBlank(systemTopicClusterName)) {
- this.systemTopicClusterName = this.rocketMQClusterName;
+ if (StringUtils.isBlank(heartbeatSyncerTopicClusterName)) {
+ this.heartbeatSyncerTopicClusterName = this.rocketMQClusterName;
}
}
@@ -324,12 +326,12 @@ public class ProxyConfig implements ConfigFile {
this.localServeAddr = localServeAddr;
}
- public String getSystemTopicClusterName() {
- return systemTopicClusterName;
+ public String getHeartbeatSyncerTopicClusterName() {
+ return heartbeatSyncerTopicClusterName;
}
- public void setSystemTopicClusterName(String systemTopicClusterName) {
- this.systemTopicClusterName = systemTopicClusterName;
+ public void setHeartbeatSyncerTopicClusterName(String heartbeatSyncerTopicClusterName) {
+ this.heartbeatSyncerTopicClusterName = heartbeatSyncerTopicClusterName;
}
public int getHeartbeatSyncerThreadPoolNums() {
@@ -348,6 +350,14 @@ public class ProxyConfig implements ConfigFile {
this.heartbeatSyncerThreadPoolQueueCapacity = heartbeatSyncerThreadPoolQueueCapacity;
}
+ public String getHeartbeatSyncerTopicName() {
+ return heartbeatSyncerTopicName;
+ }
+
+ public void setHeartbeatSyncerTopicName(String heartbeatSyncerTopicName) {
+ this.heartbeatSyncerTopicName = heartbeatSyncerTopicName;
+ }
+
public boolean isEnablePrintJstack() {
return enablePrintJstack;
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
index e0a9fd702..d08d5dfb1 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
@@ -49,7 +49,6 @@ public abstract class AbstractSystemMessageSyncer implements StartAndShutdown, M
protected static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
protected final TopicRouteService topicRouteService;
protected final AdminService adminService;
- protected final String systemResourceName;
protected final MQClientAPIFactory mqClientAPIFactory;
protected DefaultMQPushConsumer defaultMQPushConsumer;
@@ -57,25 +56,18 @@ public abstract class AbstractSystemMessageSyncer implements StartAndShutdown, M
this.topicRouteService = topicRouteService;
this.adminService = adminService;
this.mqClientAPIFactory = mqClientAPIFactory;
-
- this.systemResourceName = this.getSystemResourceName();
- }
-
- protected String getSystemResourceName() {
- ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
- return TopicValidator.SYSTEM_TOPIC_PREFIX + "proxy_" + this.getClass().getSimpleName() + "_" + proxyConfig.getProxyClusterName();
}
protected String getSystemMessageProducerId() {
- return "PID_" + this.systemResourceName;
+ return "PID_" + getBroadcastTopicName();
}
protected String getSystemMessageConsumerId() {
- return "CID_" + this.systemResourceName;
+ return "CID_" + getBroadcastTopicName();
}
protected String getBroadcastTopicName() {
- return this.systemResourceName;
+ return ConfigurationManager.getProxyConfig().getHeartbeatSyncerTopicName();
}
protected String getSubTag() {
@@ -84,7 +76,7 @@ public abstract class AbstractSystemMessageSyncer implements StartAndShutdown, M
protected String getBroadcastTopicClusterName() {
ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
- return proxyConfig.getSystemTopicClusterName();
+ return proxyConfig.getHeartbeatSyncerTopicClusterName();
}
protected int getBroadcastTopicQueueNum() {
diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java
index 3fb955a0c..95810b97c 100644
--- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java
@@ -156,6 +156,7 @@ public class GrpcBaseIT extends BaseConf {
// Set LongPollingReserveTimeInMillis to 500ms to reserve more time for IT
ConfigurationManager.getProxyConfig().setLongPollingReserveTimeInMillis(500);
ConfigurationManager.getProxyConfig().setRocketMQClusterName(brokerController1.getBrokerConfig().getBrokerClusterName());
+ ConfigurationManager.getProxyConfig().setHeartbeatSyncerTopicClusterName(brokerController1.getBrokerConfig().getBrokerClusterName());
ConfigurationManager.getProxyConfig().setMinInvisibleTimeMillsForRecv(3);
}
[rocketmq] 17/26: [ISSUE #5406] Fix brokerName for SEND_MESSAGE_V2
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 274a531321bb10afb2f0bfb0b8d837ced2d8e7af
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Fri Nov 18 15:46:17 2022 +0800
[ISSUE #5406] Fix brokerName for SEND_MESSAGE_V2
---
.../remoting/activity/AbstractRemotingActivity.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
index 54ef7bfa7..650c38614 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
@@ -25,6 +25,7 @@ import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.common.ProxyException;
@@ -44,6 +45,7 @@ public abstract class AbstractRemotingActivity implements NettyRequestProcessor
protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
protected final MessagingProcessor messagingProcessor;
protected static final String BROKER_NAME_FIELD = "bname";
+ protected static final String BROKER_NAME_FIELD_FOR_SEND_MESSAGE_V2 = "n";
private static final Map<ProxyExceptionCode, Integer> PROXY_EXCEPTION_RESPONSE_CODE_MAP = new HashMap<ProxyExceptionCode, Integer>() {
{
put(ProxyExceptionCode.FORBIDDEN, ResponseCode.NO_PERMISSION);
@@ -61,11 +63,20 @@ public abstract class AbstractRemotingActivity implements NettyRequestProcessor
protected RemotingCommand request(ChannelHandlerContext ctx, RemotingCommand request,
ProxyContext context, long timeoutMillis) throws Exception {
- if (request.getExtFields().get(BROKER_NAME_FIELD) == null) {
- return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,
- "Request doesn't have field bname");
+ String brokerName;
+ if (request.getCode() == RequestCode.SEND_MESSAGE_V2) {
+ if (request.getExtFields().get(BROKER_NAME_FIELD_FOR_SEND_MESSAGE_V2) == null) {
+ return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,
+ "Request doesn't have field bname");
+ }
+ brokerName = request.getExtFields().get(BROKER_NAME_FIELD_FOR_SEND_MESSAGE_V2);
+ } else {
+ if (request.getExtFields().get(BROKER_NAME_FIELD) == null) {
+ return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,
+ "Request doesn't have field bname");
+ }
+ brokerName = request.getExtFields().get(BROKER_NAME_FIELD);
}
- String brokerName = request.getExtFields().get(BROKER_NAME_FIELD);
if (request.isOnewayRPC()) {
messagingProcessor.requestOneway(context, brokerName, request, timeoutMillis);
return null;
[rocketmq] 13/26: [ISSUE #5485] polish channel management
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 96bab46bd4d3cf3aaa9b2c07acec15907821048b
Author: kaiyi.lk <ka...@alibaba-inc.com>
AuthorDate: Thu Nov 17 14:20:28 2022 +0800
[ISSUE #5485] polish channel management
---
.../rocketmq/proxy/common/utils/FutureUtils.java | 6 ++++
.../proxy/remoting/channel/RemotingChannel.java | 23 ++++++---------
.../remoting/channel/RemotingChannelManager.java | 33 +++++++++++-----------
.../proxy/remoting/common/RemotingConverter.java | 23 ---------------
.../proxy/service/admin/DefaultAdminService.java | 6 +++-
5 files changed, 36 insertions(+), 55 deletions(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java
index 2e194a8cb..ea50e64ee 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java
@@ -37,4 +37,10 @@ public class FutureUtils {
public static <T> CompletableFuture<T> addExecutor(CompletableFuture<T> future, ExecutorService executor) {
return appendNextFuture(future, new CompletableFuture<>(), executor);
}
+
+ public static <T> CompletableFuture<T> completeExceptionally(Throwable t) {
+ CompletableFuture<T> future = new CompletableFuture<>();
+ future.completeExceptionally(t);
+ return future;
+ }
}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
index 8b4832cad..806b35de2 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
@@ -38,6 +38,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
+import org.apache.rocketmq.proxy.common.utils.FutureUtils;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter;
import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
@@ -132,54 +133,48 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver
protected CompletableFuture<Void> processGetConsumerRunningInfo(RemotingCommand command,
GetConsumerRunningInfoRequestHeader header,
CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> responseFuture) {
- CompletableFuture<Void> writeFuture = new CompletableFuture<>();
try {
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, header);
- return this.remotingProxyOutClient.invokeToClient(this.parent(), request, DEFAULT_MQ_CLIENT_TIMEOUT)
- .thenApply(response -> {
+ this.remotingProxyOutClient.invokeToClient(this.parent(), request, DEFAULT_MQ_CLIENT_TIMEOUT)
+ .thenAccept(response -> {
if (response.getCode() == ResponseCode.SUCCESS) {
ConsumerRunningInfo consumerRunningInfo = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class);
responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", consumerRunningInfo));
- return null;
}
String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark());
RuntimeException e = new RuntimeException(errMsg);
responseFuture.completeExceptionally(e);
- throw e;
});
+ return CompletableFuture.completedFuture(null);
} catch (Throwable t) {
responseFuture.completeExceptionally(t);
- writeFuture.completeExceptionally(t);
+ return FutureUtils.completeExceptionally(t);
}
- return writeFuture;
}
@Override
protected CompletableFuture<Void> processConsumeMessageDirectly(RemotingCommand command,
ConsumeMessageDirectlyResultRequestHeader header, MessageExt messageExt,
CompletableFuture<ProxyRelayResult<ConsumeMessageDirectlyResult>> responseFuture) {
- CompletableFuture<Void> writeFuture = new CompletableFuture<>();
try {
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, header);
request.setBody(RemotingConverter.getInstance().convertMsgToBytes(messageExt));
- return this.remotingProxyOutClient.invokeToClient(this.parent(), request, DEFAULT_MQ_CLIENT_TIMEOUT)
- .thenApply(response -> {
+ this.remotingProxyOutClient.invokeToClient(this.parent(), request, DEFAULT_MQ_CLIENT_TIMEOUT)
+ .thenAccept(response -> {
if (response.getCode() == ResponseCode.SUCCESS) {
ConsumeMessageDirectlyResult result = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class);
responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", result));
- return null;
}
String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark());
RuntimeException e = new RuntimeException(errMsg);
responseFuture.completeExceptionally(e);
- throw e;
});
+ return CompletableFuture.completedFuture(null);
} catch (Throwable t) {
responseFuture.completeExceptionally(t);
- writeFuture.completeExceptionally(t);
+ return FutureUtils.completeExceptionally(t);
}
- return writeFuture;
}
public String getClientId() {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
index bdc6457e7..d47884b61 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannelManager.java
@@ -57,10 +57,6 @@ public class RemotingChannelManager implements StartAndShutdown {
return prefix + group;
}
- protected String getGroupFromKey(String key) {
- return key.substring(1);
- }
-
public RemotingChannel createProducerChannel(Channel channel, String group, String clientId) {
return createChannel(channel, buildProducerKey(group), clientId, Collections.emptySet());
}
@@ -80,14 +76,6 @@ public class RemotingChannelManager implements StartAndShutdown {
return getChannel(group, channel);
}
- public RemotingChannel getConsumerChannel(String group, Channel channel) {
- return getChannel(buildConsumerKey(group), channel);
- }
-
- public RemotingChannel getProducerChannel(String group, Channel channel) {
- return getChannel(buildProducerKey(group), channel);
- }
-
protected RemotingChannel getChannel(String group, Channel channel) {
Map<Channel, RemotingChannel> clientIdChannelMap = this.groupChannelMap.get(group);
if (clientIdChannelMap == null) {
@@ -98,10 +86,9 @@ public class RemotingChannelManager implements StartAndShutdown {
public Set<RemotingChannel> removeChannel(Channel channel) {
Set<RemotingChannel> removedChannelSet = new HashSet<>();
- for (Map.Entry<String, Map<Channel /* raw channel */, RemotingChannel>> entry : groupChannelMap.entrySet()) {
- Map<Channel /* raw channel */, RemotingChannel> channelMap = entry.getValue();
-
- RemotingChannel remotingChannel = channelMap.remove(channel);
+ Set<String> groupKeySet = groupChannelMap.keySet();
+ for (String group : groupKeySet) {
+ RemotingChannel remotingChannel = removeChannel(group, channel);
if (remotingChannel != null) {
removedChannelSet.add(remotingChannel);
}
@@ -121,7 +108,7 @@ public class RemotingChannelManager implements StartAndShutdown {
AtomicReference<RemotingChannel> channelRef = new AtomicReference<>();
this.groupChannelMap.computeIfPresent(group, (groupKey, channelMap) -> {
- channelRef.set(channelMap.remove(channel));
+ channelRef.set(channelMap.remove(getOrgRawChannel(channel)));
if (channelMap.isEmpty()) {
return null;
}
@@ -130,6 +117,18 @@ public class RemotingChannelManager implements StartAndShutdown {
return channelRef.get();
}
+ /**
+ * to get the org channel pass by nettyRemotingServer
+ * @param channel
+ * @return
+ */
+ protected Channel getOrgRawChannel(Channel channel) {
+ if (channel instanceof RemotingChannel) {
+ return channel.parent();
+ }
+ return channel;
+ }
+
@Override
public void shutdown() throws Exception {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java
index 55af1ff19..a08abbba2 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/common/RemotingConverter.java
@@ -17,8 +17,6 @@
package org.apache.rocketmq.proxy.remoting.common;
-import java.nio.ByteBuffer;
-import java.util.List;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
@@ -42,27 +40,6 @@ public class RemotingConverter {
return instance;
}
- public byte[] convertMsgToBytes(List<MessageExt> msgList) {
- // set response body
- byte[][] msgBufferList = new byte[msgList.size()][];
- int bodyTotalSize = 0;
- for (int i = 0; i < msgList.size(); i++) {
- try {
- msgBufferList[i] = convertMsgToBytes(msgList.get(i));
- bodyTotalSize += msgBufferList[i].length;
- } catch (Exception e) {
- log.error("messageToByteBuffer UnsupportedEncodingException", e);
- }
- }
-
- ByteBuffer body = ByteBuffer.allocate(bodyTotalSize);
- for (byte[] bb : msgBufferList) {
- body.put(bb);
- }
-
- return body.array();
- }
-
public byte[] convertMsgToBytes(final MessageExt msg) throws Exception {
// change to 0 for recalculate storeSize
msg.setStoreSize(0);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
index 4f3b407d6..e94c0879b 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java
@@ -115,7 +115,11 @@ public class DefaultAdminService implements AdminService {
continue;
}
- this.getClient().createTopic(addr, TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, topicConfig, Duration.ofSeconds(3).toMillis());
+ try {
+ this.getClient().createTopic(addr, TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, topicConfig, Duration.ofSeconds(3).toMillis());
+ } catch (Exception e) {
+ log.error("create topic on broker failed. topic:{}, broker:{}", topicConfig, addr, e);
+ }
}
if (examineTopic) {
[rocketmq] 16/26: [ISSUE #5392] Remove unused method
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 561fc2e73253ac36c055bc0f26fa7b44e1f7d052
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Thu Nov 17 17:55:43 2022 +0800
[ISSUE #5392] Remove unused method
---
.../protocol/header/SendMessageRequestHeader.java | 74 ----------------------
1 file changed, 74 deletions(-)
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java
index be7d2038a..17ce51263 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java
@@ -21,7 +21,6 @@
package org.apache.rocketmq.remoting.protocol.header;
import com.google.common.base.MoreObjects;
-import java.util.HashMap;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -191,79 +190,6 @@ public class SendMessageRequestHeader extends TopicQueueRequestHeader {
return requestHeader;
}
- public static SendMessageRequestHeaderV2 decodeSendMessageHeaderV2(RemotingCommand request)
- throws RemotingCommandException {
- SendMessageRequestHeaderV2 r = new SendMessageRequestHeaderV2();
- HashMap<String, String> fields = request.getExtFields();
- if (fields == null) {
- throw new RemotingCommandException("the ext fields is null");
- }
-
- String s = fields.get("a");
- checkNotNull(s, "the custom field <a> is null");
- r.setA(s);
-
- s = fields.get("b");
- checkNotNull(s, "the custom field <b> is null");
- r.setB(s);
-
- s = fields.get("c");
- checkNotNull(s, "the custom field <c> is null");
- r.setC(s);
-
- s = fields.get("d");
- checkNotNull(s, "the custom field <d> is null");
- r.setD(Integer.parseInt(s));
-
- s = fields.get("e");
- checkNotNull(s, "the custom field <e> is null");
- r.setE(Integer.parseInt(s));
-
- s = fields.get("f");
- checkNotNull(s, "the custom field <f> is null");
- r.setF(Integer.parseInt(s));
-
- s = fields.get("g");
- checkNotNull(s, "the custom field <g> is null");
- r.setG(Long.parseLong(s));
-
- s = fields.get("h");
- checkNotNull(s, "the custom field <h> is null");
- r.setH(Integer.parseInt(s));
-
- s = fields.get("i");
- if (s != null) {
- r.setI(s);
- }
-
- s = fields.get("j");
- if (s != null) {
- r.setJ(Integer.parseInt(s));
- }
-
- s = fields.get("k");
- if (s != null) {
- r.setK(Boolean.parseBoolean(s));
- }
-
- s = fields.get("l");
- if (s != null) {
- r.setL(Integer.parseInt(s));
- }
-
- s = fields.get("m");
- if (s != null) {
- r.setM(Boolean.parseBoolean(s));
- }
- return r;
- }
-
- private static void checkNotNull(String s, String msg) throws RemotingCommandException {
- if (s == null) {
- throw new RemotingCommandException(msg);
- }
- }
-
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
[rocketmq] 25/26: [ISSUE #5485] Fix merge code issue
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 373c1bd13ef7148bc9d04db155bed77d98e00429
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Fri Dec 9 10:23:23 2022 +0800
[ISSUE #5485] Fix merge code issue
---
proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
index 4d5084cfd..9c833ba8a 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
@@ -1173,7 +1173,6 @@ public class ProxyConfig implements ConfigFile {
public void setChannelExpiredTimeout(long channelExpiredTimeout) {
this.channelExpiredTimeout = channelExpiredTimeout;
}
-}
public void setEnableRemotingLocalProxyGrpc(boolean enableRemotingLocalProxyGrpc) {
this.enableRemotingLocalProxyGrpc = enableRemotingLocalProxyGrpc;
[rocketmq] 23/26: [ISSUE #5485] Remove ConsumerManagerInterface and ProducerManagerInterface
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouxzhan pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 8c9a9d36b844390cccc2ab083ed63d5b222e256c
Author: zhouxiang <zh...@alibaba-inc.com>
AuthorDate: Tue Dec 6 16:50:59 2022 +0800
[ISSUE #5485] Remove ConsumerManagerInterface and ProducerManagerInterface
---
.../rocketmq/broker/client/ConsumerManager.java | 13 +----
.../broker/client/ConsumerManagerInterface.java | 60 ----------------------
.../rocketmq/broker/client/ProducerManager.java | 12 +----
.../broker/client/ProducerManagerInterface.java | 44 ----------------
.../rocketmq/proxy/service/ServiceManager.java | 8 +--
.../service/client/ClusterConsumerManager.java | 9 ++--
.../proxy/service/sysmessage/HeartbeatSyncer.java | 12 ++---
.../service/sysmessage/HeartbeatSyncerTest.java | 4 +-
8 files changed, 18 insertions(+), 144 deletions(-)
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
index a70e8579e..787dcdbd2 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
@@ -38,7 +38,7 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
-public class ConsumerManager implements ConsumerManagerInterface {
+public class ConsumerManager {
private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final ConcurrentMap<String, ConsumerGroupInfo> consumerTable =
new ConcurrentHashMap<>(1024);
@@ -64,7 +64,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
this.subscriptionExpiredTimeout = brokerConfig.getSubscriptionExpiredTimeout();
}
- @Override
public ClientChannelInfo findChannel(final String group, final String clientId) {
ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
if (consumerGroupInfo != null) {
@@ -73,7 +72,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
return null;
}
- @Override
public ClientChannelInfo findChannel(final String group, final Channel channel) {
ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
if (consumerGroupInfo != null) {
@@ -82,7 +80,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
return null;
}
- @Override
public SubscriptionData findSubscriptionData(final String group, final String topic) {
return findSubscriptionData(group, topic, true);
}
@@ -110,7 +107,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
return this.consumerTable;
}
- @Override
public ConsumerGroupInfo getConsumerGroupInfo(final String group) {
return getConsumerGroupInfo(group, false);
}
@@ -123,7 +119,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
return consumerGroupInfo;
}
- @Override
public int findSubscriptionDataCount(final String group) {
ConsumerGroupInfo consumerGroupInfo = this.getConsumerGroupInfo(group);
if (consumerGroupInfo != null) {
@@ -133,7 +128,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
return 0;
}
- @Override
public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
boolean removed = false;
Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
@@ -178,7 +172,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
isNotifyConsumerIdsChangedEnable, true);
}
- @Override
public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription) {
@@ -214,7 +207,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
return r1 || r2;
}
- @Override
public void unregisterConsumer(final String group, final ClientChannelInfo clientChannelInfo,
boolean isNotifyConsumerIdsChangedEnable) {
ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
@@ -260,7 +252,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
}
}
- @Override
public void scanNotActiveChannel() {
Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
while (it.hasNext()) {
@@ -295,7 +286,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
removeExpireConsumerGroupInfo();
}
- @Override
public HashSet<String> queryTopicConsumeByWho(final String topic) {
HashSet<String> groups = new HashSet<>();
Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
@@ -310,7 +300,6 @@ public class ConsumerManager implements ConsumerManagerInterface {
return groups;
}
- @Override
public void appendConsumerIdsChangeListener(ConsumerIdsChangeListener listener) {
consumerIdsChangeListenerList.add(listener);
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java
deleted file mode 100644
index 6998f60e7..000000000
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManagerInterface.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.broker.client;
-
-import io.netty.channel.Channel;
-import java.util.Set;
-import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
-import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
-import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
-
-public interface ConsumerManagerInterface {
-
- ClientChannelInfo findChannel(String group, String clientId);
-
- ClientChannelInfo findChannel(String group, Channel channel);
-
- SubscriptionData findSubscriptionData(String group, String topic);
-
- ConsumerGroupInfo getConsumerGroupInfo(String group);
-
- int findSubscriptionDataCount(String group);
-
- boolean doChannelCloseEvent(String remoteAddr, Channel channel);
-
- default boolean registerConsumer(String group, ClientChannelInfo clientChannelInfo,
- ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
- Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {
- return registerConsumer(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList,
- isNotifyConsumerIdsChangedEnable, true);
- }
-
- boolean registerConsumer(String group, ClientChannelInfo clientChannelInfo,
- ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
- Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription);
-
- void unregisterConsumer(String group, ClientChannelInfo clientChannelInfo,
- boolean isNotifyConsumerIdsChangedEnable);
-
- void scanNotActiveChannel();
-
- Set<String> queryTopicConsumeByWho(String topic);
-
- void appendConsumerIdsChangeListener(ConsumerIdsChangeListener listener);
-}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
index a3ed9c590..52d67bf28 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
@@ -35,7 +35,7 @@ import org.apache.rocketmq.remoting.protocol.body.ProducerInfo;
import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
-public class ProducerManager implements ProducerManagerInterface {
+public class ProducerManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;
private static final int GET_AVAILABLE_CHANNEL_RETRY_COUNT = 3;
@@ -54,12 +54,10 @@ public class ProducerManager implements ProducerManagerInterface {
this.brokerStatsManager = brokerStatsManager;
}
- @Override
public int groupSize() {
return this.groupChannelTable.size();
}
- @Override
public boolean groupOnline(String group) {
Map<Channel, ClientChannelInfo> channels = this.groupChannelTable.get(group);
return channels != null && !channels.isEmpty();
@@ -69,7 +67,6 @@ public class ProducerManager implements ProducerManagerInterface {
return groupChannelTable;
}
- @Override
public ProducerTableInfo getProducerTable() {
Map<String, List<ProducerInfo>> map = new HashMap<>();
for (String group : this.groupChannelTable.keySet()) {
@@ -97,7 +94,6 @@ public class ProducerManager implements ProducerManagerInterface {
return new ProducerTableInfo(map);
}
- @Override
public void scanNotActiveChannel() {
Iterator<Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>>> iterator = this.groupChannelTable.entrySet().iterator();
@@ -133,7 +129,6 @@ public class ProducerManager implements ProducerManagerInterface {
}
}
- @Override
public synchronized boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
boolean removed = false;
if (channel != null) {
@@ -165,7 +160,6 @@ public class ProducerManager implements ProducerManagerInterface {
return removed;
}
- @Override
public synchronized void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) {
ClientChannelInfo clientChannelInfoFound = null;
@@ -189,7 +183,6 @@ public class ProducerManager implements ProducerManagerInterface {
}
}
- @Override
public synchronized void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) {
ConcurrentHashMap<Channel, ClientChannelInfo> channelTable = this.groupChannelTable.get(group);
if (null != channelTable && !channelTable.isEmpty()) {
@@ -209,7 +202,6 @@ public class ProducerManager implements ProducerManagerInterface {
}
}
- @Override
public Channel getAvailableChannel(String groupId) {
if (groupId == null) {
return null;
@@ -250,7 +242,6 @@ public class ProducerManager implements ProducerManagerInterface {
return lastActiveChannel;
}
- @Override
public Channel findChannel(String clientId) {
return clientChannelTable.get(clientId);
}
@@ -266,7 +257,6 @@ public class ProducerManager implements ProducerManagerInterface {
}
}
- @Override
public void appendProducerChangeListener(ProducerChangeListener producerChangeListener) {
producerChangeListenerList.add(producerChangeListener);
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java
deleted file mode 100644
index 5e2e7e5b0..000000000
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManagerInterface.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.broker.client;
-
-import io.netty.channel.Channel;
-import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;
-
-public interface ProducerManagerInterface {
-
- int groupSize();
-
- boolean groupOnline(String group);
-
- ProducerTableInfo getProducerTable();
-
- void scanNotActiveChannel();
-
- boolean doChannelCloseEvent(String remoteAddr, Channel channel);
-
- void registerProducer(String group, ClientChannelInfo clientChannelInfo);
-
- void unregisterProducer(String group, ClientChannelInfo clientChannelInfo);
-
- Channel getAvailableChannel(String groupId);
-
- Channel findChannel(String clientId);
-
- void appendProducerChangeListener(ProducerChangeListener producerChangeListener);
-}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java
index ce84832ca..bfa2ed963 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java
@@ -16,8 +16,8 @@
*/
package org.apache.rocketmq.proxy.service;
-import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
-import org.apache.rocketmq.broker.client.ProducerManagerInterface;
+import org.apache.rocketmq.broker.client.ConsumerManager;
+import org.apache.rocketmq.broker.client.ProducerManager;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
import org.apache.rocketmq.proxy.service.admin.AdminService;
import org.apache.rocketmq.proxy.service.message.MessageService;
@@ -31,9 +31,9 @@ public interface ServiceManager extends StartAndShutdown {
TopicRouteService getTopicRouteService();
- ProducerManagerInterface getProducerManager();
+ ProducerManager getProducerManager();
- ConsumerManagerInterface getConsumerManager();
+ ConsumerManager getConsumerManager();
TransactionService getTransactionService();
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
index 3a98b5ee1..94f4c5232 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/client/ClusterConsumerManager.java
@@ -21,18 +21,17 @@ import java.util.Set;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
import org.apache.rocketmq.broker.client.ConsumerManager;
-import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
-import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
-import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.proxy.common.StartAndShutdown;
import org.apache.rocketmq.proxy.service.admin.AdminService;
import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
import org.apache.rocketmq.proxy.service.route.TopicRouteService;
import org.apache.rocketmq.proxy.service.sysmessage.HeartbeatSyncer;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
-public class ClusterConsumerManager extends ConsumerManager implements ConsumerManagerInterface, StartAndShutdown {
+public class ClusterConsumerManager extends ConsumerManager implements StartAndShutdown {
protected HeartbeatSyncer heartbeatSyncer;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
index 041cbcee6..9e333902e 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
@@ -28,14 +28,11 @@ import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
-import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
+import org.apache.rocketmq.broker.client.ConsumerManager;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
-import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
@@ -44,15 +41,18 @@ import org.apache.rocketmq.proxy.processor.channel.RemoteChannel;
import org.apache.rocketmq.proxy.service.admin.AdminService;
import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory;
import org.apache.rocketmq.proxy.service.route.TopicRouteService;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
protected ThreadPoolExecutor threadPoolExecutor;
- protected ConsumerManagerInterface consumerManager;
+ protected ConsumerManager consumerManager;
protected final Map<String /* channelId as longText */, RemoteChannel> remoteChannelMap = new ConcurrentHashMap<>();
public HeartbeatSyncer(TopicRouteService topicRouteService, AdminService adminService,
- ConsumerManagerInterface consumerManager, MQClientAPIFactory mqClientAPIFactory) {
+ ConsumerManager consumerManager, MQClientAPIFactory mqClientAPIFactory) {
super(topicRouteService, adminService, mqClientAPIFactory);
this.consumerManager = consumerManager;
this.init();
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
index 45e3942d6..95152186d 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
@@ -34,7 +34,7 @@ import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
-import org.apache.rocketmq.broker.client.ConsumerManagerInterface;
+import org.apache.rocketmq.broker.client.ConsumerManager;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.client.producer.SendStatus;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
@@ -94,7 +94,7 @@ public class HeartbeatSyncerTest extends InitConfigTest {
@Mock
private AdminService adminService;
@Mock
- private ConsumerManagerInterface consumerManager;
+ private ConsumerManager consumerManager;
@Mock
private MQClientAPIFactory mqClientAPIFactory;
@Mock