You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by ji...@apache.org on 2022/07/08 07:04:48 UTC
[rocketmq] 01/01: Merge branch '5.0.0-beta' into develop
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch develop-merge-5.0.0
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 3fadc8f30c898f9ddd6730ba4ac428894a1c1077
Merge: 418a5b2ea ef37465e5
Author: RongtongJin <ji...@mails.ucas.ac.cn>
AuthorDate: Fri Jul 8 15:03:54 2022 +0800
Merge branch '5.0.0-beta' into develop
# Conflicts:
# acl/pom.xml
# broker/pom.xml
# broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
# broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
# broker/src/main/java/org/apache/rocketmq/broker/plugin/MessageStoreFactory.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/ForwardRequestProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java
# broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java
# client/pom.xml
# client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
# client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
# client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
# client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
# client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
# client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java
# common/pom.xml
# common/src/main/java/org/apache/rocketmq/common/MQVersion.java
# common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
# common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
# common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
# common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
# common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
# common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
# common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java
# distribution/pom.xml
# example/pom.xml
# filter/pom.xml
# logging/pom.xml
# logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java
# namesrv/pom.xml
# namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java
# namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
# namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
# openmessaging/pom.xml
# pom.xml
# remoting/pom.xml
# remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
# remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
# remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
# remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java
# srvutil/pom.xml
# store/pom.xml
# store/src/main/java/org/apache/rocketmq/store/CommitLog.java
# store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
# store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
# store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java
# store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
# store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java
# store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
# store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
# store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
# store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
# store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreCleanFilesTest.java
# store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
# store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java
# test/pom.xml
# tools/pom.xml
# tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
# tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
# tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
# tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
# tools/src/main/java/org/apache/rocketmq/tools/command/topic/DeleteTopicSubCommand.java
.travis.yml | 9 +-
acl/pom.xml | 2 +-
.../org/apache/rocketmq/acl/common/AclUtils.java | 39 +
.../apache/rocketmq/acl/common/AclUtilsTest.java | 21 +
acl/src/test/resources/conf/plain_acl.yml | 5 -
broker/pom.xml | 10 +-
.../apache/rocketmq/broker/BrokerController.java | 1761 ++++++++++++++-----
.../rocketmq/broker/BrokerPathConfigHelper.java | 12 +
.../rocketmq/broker/BrokerPreOnlineService.java | 277 +++
.../org/apache/rocketmq/broker/BrokerStartup.java | 58 +-
.../org/apache/rocketmq/broker/ShutdownHook.java | 15 +-
.../broker/client/ClientHousekeepingService.java | 12 +-
.../rocketmq/broker/client/ConsumerGroupInfo.java | 25 +-
.../broker/client/ConsumerIdsChangeListener.java | 2 +
.../rocketmq/broker/client/ConsumerManager.java | 42 +-
.../client/DefaultConsumerIdsChangeListener.java | 68 +-
.../rocketmq/broker/client/ProducerManager.java | 23 +-
.../rocketmq/broker/client/net/Broker2Client.java | 8 +-
.../client/rebalance/RebalanceLockManager.java | 99 +-
.../broker/dledger/DLedgerRoleChangeHandler.java | 90 +-
.../rocketmq/broker/failover/EscapeBridge.java | 268 +++
.../broker/filtersrv/FilterServerManager.java | 6 +-
.../rocketmq/broker/latency/BrokerFastFailure.java | 44 +-
.../broker/loadbalance/AssignmentManager.java | 140 ++
.../loadbalance/MessageRequestModeManager.java | 95 +
.../longpolling/LmqPullRequestHoldService.java | 3 +
.../broker/longpolling/ManyPullRequest.java | 4 +
.../broker/longpolling/NotificationRequest.java | 57 +
.../longpolling/NotifyMessageArrivingListener.java | 12 +-
.../rocketmq/broker/longpolling/PopRequest.java | 89 +
.../broker/longpolling/PullRequestHoldService.java | 36 +-
.../broker/mqtrace/AbortProcessException.java | 69 +
.../broker/mqtrace/ConsumeMessageContext.java | 77 +-
.../broker/mqtrace/SendMessageContext.java | 102 +-
.../broker/offset/ConsumerOffsetManager.java | 85 +-
.../broker/offset/ConsumerOrderInfoManager.java | 426 +++++
.../apache/rocketmq/broker/out/BrokerOuterAPI.java | 750 +++++++-
.../broker/pagecache/OneMessageTransfer.java | 1 +
.../broker/plugin/AbstractPluginMessageStore.java | 292 +++-
.../broker/plugin/BrokerAttachedPlugin.java | 74 +
.../broker/plugin/MessageStoreFactory.java | 15 +-
.../broker/plugin/MessageStorePluginContext.java | 16 +-
.../broker/plugin/PullMessageResultHandler.java | 53 +
.../processor/AbstractSendMessageProcessor.java | 457 ++++-
.../broker/processor/AckMessageProcessor.java | 214 +++
.../broker/processor/AdminBrokerProcessor.java | 992 +++++++++--
.../processor/ChangeInvisibleTimeProcessor.java | 206 +++
.../broker/processor/ClientManageProcessor.java | 52 +-
.../broker/processor/ConsumerManageProcessor.java | 188 +-
.../processor/DefaultPullMessageResultHandler.java | 244 +++
.../broker/processor/EndTransactionProcessor.java | 10 +-
.../broker/processor/ForwardRequestProcessor.java | 4 +-
.../broker/processor/NotificationProcessor.java | 318 ++++
.../broker/processor/PeekMessageProcessor.java | 271 +++
.../broker/processor/PollingInfoProcessor.java | 119 ++
.../broker/processor/PopBufferMergeService.java | 776 +++++++++
.../broker/processor/PopMessageProcessor.java | 983 +++++++++++
.../broker/processor/PopReviveService.java | 488 ++++++
.../broker/processor/PullMessageProcessor.java | 571 +++---
.../broker/processor/QueryAssignmentProcessor.java | 324 ++++
.../broker/processor/QueryMessageProcessor.java | 15 +-
.../broker/processor/ReplyMessageProcessor.java | 13 +-
.../broker/processor/SendMessageCallback.java | 22 +-
.../broker/processor/SendMessageProcessor.java | 603 +++----
.../schedule/DelayOffsetSerializeWrapper.java | 13 +-
.../broker}/schedule/ScheduleMessageService.java | 183 +-
.../rocketmq/broker/slave/SlaveSynchronize.java | 73 +-
.../subscription/SubscriptionGroupManager.java | 112 +-
.../rocketmq/broker/topic/TopicConfigManager.java | 287 ++-
.../topic/TopicQueueMappingCleanService.java | 337 ++++
.../broker/topic/TopicQueueMappingManager.java | 259 +++
.../AbstractTransactionalMessageCheckListener.java | 45 +-
.../broker/transaction/OperationResult.java | 8 +-
.../TransactionalMessageCheckService.java | 3 +
.../transaction/TransactionalMessageService.java | 2 +-
.../transaction/jdbc/JDBCTransactionStore.java | 242 ---
.../jdbc/JDBCTransactionStoreConfig.java | 57 -
.../DefaultTransactionalMessageCheckListener.java | 2 +-
.../queue/TransactionalMessageBridge.java | 2 +-
.../queue/TransactionalMessageServiceImpl.java | 2 +-
.../org/apache/rocketmq/broker/util/HookUtils.java | 164 ++
.../OperationResult.java => util/MsgUtil.java} | 35 +-
.../rocketmq/broker/BrokerControllerTest.java | 3 +-
.../apache/rocketmq/broker/BrokerOuterAPITest.java | 61 +-
.../apache/rocketmq/broker/BrokerStartupTest.java | 20 +-
.../broker/filter/MessageStoreWithFilterTest.java | 56 +-
.../broker/offset/ConsumerOffsetManagerTest.java | 55 +
.../broker/pagecache/OneMessageTransferTest.java | 8 +-
...essorTest.java => AckMessageProcessorTest.java} | 97 +-
.../broker/processor/AdminBrokerProcessorTest.java | 107 +-
....java => ChangeInvisibleTimeProcessorTest.java} | 96 +-
.../processor/ClientManageProcessorTest.java | 1 -
.../processor/ConsumerManageProcessorTest.java | 91 +
.../processor/EndTransactionProcessorTest.java | 2 +-
.../processor/PopBufferMergeServiceTest.java | 120 ++
...essorTest.java => PopMessageProcessorTest.java} | 163 +-
.../broker/processor/PullMessageProcessorTest.java | 28 +-
.../processor/QueryAssignmentProcessorTest.java | 227 +++
.../processor/ReplyMessageProcessorTest.java | 2 +-
.../broker/processor/SendMessageProcessorTest.java | 217 ++-
.../schedule/ScheduleMessageServiceTest.java | 145 +-
.../broker/substription/ForbiddenTest.java | 64 +
.../broker/topic/TopicConfigManagerTest.java | 324 ++++
.../broker/topic/TopicQueueMappingManagerTest.java | 112 ++
...faultTransactionalMessageCheckListenerTest.java | 2 +-
.../queue/TransactionalMessageBridgeTest.java | 2 +-
.../queue/TransactionalMessageServiceImplTest.java | 2 +-
.../rocketmq/broker/util/ServiceProviderTest.java | 7 +-
.../util/TransactionalMessageServiceImpl.java | 2 +-
client/pom.xml | 11 +-
.../java/org/apache/rocketmq/client/MQAdmin.java | 14 +-
.../rocketmq/client/consumer/AckCallback.java | 10 +-
.../apache/rocketmq/client/consumer/AckResult.java | 40 +-
.../apache/rocketmq/client/consumer/AckStatus.java | 17 +-
.../client/consumer/DefaultMQPullConsumer.java | 14 +-
.../client/consumer/DefaultMQPushConsumer.java | 70 +-
.../rocketmq/client/consumer/MQPullConsumer.java | 7 +
.../rocketmq/client/consumer/PopCallback.java | 13 +-
.../consumer/{PullResult.java => PopResult.java} | 59 +-
.../apache/rocketmq/client/consumer/PopStatus.java | 29 +-
.../rocketmq/client/consumer/PullResult.java | 1 +
.../AbstractAllocateMessageQueueStrategy.java | 2 +-
.../rebalance/AllocateMessageQueueAveragely.java | 10 +
.../AllocateMessageQueueAveragelyByCircle.java | 10 +
.../consumer/store/RemoteBrokerOffsetStore.java | 31 +-
.../client/exception/MQBrokerException.java | 6 +
.../client/exception/MQRedirectException.java | 26 +-
.../client/exception/OffsetNotFoundException.java | 23 +-
.../rocketmq/client/impl/BaseInvokeCallback.java | 26 +-
.../client/impl/ClientRemotingProcessor.java | 3 +-
.../apache/rocketmq/client/impl/MQAdminImpl.java | 64 +-
.../rocketmq/client/impl/MQClientAPIImpl.java | 632 ++++++-
.../ConsumeMessageConcurrentlyService.java | 11 +-
.../consumer/ConsumeMessageOrderlyService.java | 14 +-
...a => ConsumeMessagePopConcurrentlyService.java} | 260 +--
.../consumer/ConsumeMessagePopOrderlyService.java | 408 +++++
.../impl/consumer/ConsumeMessageService.java | 5 +
.../impl/consumer/DefaultLitePullConsumerImpl.java | 45 +-
.../impl/consumer/DefaultMQPullConsumerImpl.java | 43 +-
.../impl/consumer/DefaultMQPushConsumerImpl.java | 412 ++++-
.../client/impl/consumer/MessageQueueLock.java | 29 +-
.../client/impl/consumer/MessageRequest.java | 10 +-
.../client/impl/consumer/PopProcessQueue.java | 84 +
.../consumer/{PullRequest.java => PopRequest.java} | 71 +-
.../client/impl/consumer/PullAPIWrapper.java | 141 +-
.../client/impl/consumer/PullMessageService.java | 47 +-
.../rocketmq/client/impl/consumer/PullRequest.java | 8 +-
.../client/impl/consumer/PullResultExt.java | 12 +
.../client/impl/consumer/RebalanceImpl.java | 479 ++++-
.../impl/consumer/RebalanceLitePullImpl.java | 25 +-
.../client/impl/consumer/RebalancePullImpl.java | 26 +-
.../client/impl/consumer/RebalancePushImpl.java | 71 +-
.../client/impl/factory/MQClientInstance.java | 174 +-
.../impl/producer/DefaultMQProducerImpl.java | 20 +-
.../rocketmq/client/latency/MQFaultStrategy.java | 2 +-
.../client/producer/DefaultMQProducer.java | 16 +-
.../rocketmq/client/producer/SendResult.java | 9 +
.../consumer/DefaultLitePullConsumerTest.java | 32 +-
.../client/consumer/DefaultMQPushConsumerTest.java | 83 +-
.../store/RemoteBrokerOffsetStoreTest.java | 10 +-
.../rocketmq/client/impl/MQClientAPIImplTest.java | 463 ++++-
.../consumer/DefaultMQPushConsumerImplTest.java | 63 +
.../impl/consumer/RebalancePushImplTest.java | 13 -
.../client/impl/factory/MQClientInstanceTest.java | 1 +
.../client/producer/DefaultMQProducerTest.java | 2 +-
.../selector/SelectMessageQueueRetryTest.java | 1 -
.../powermock/extensions/configuration.properties | 16 +
common/pom.xml | 6 +-
...uestHeader.java => AbstractBrokerRunnable.java} | 31 +-
.../org/apache/rocketmq/common/BrokerConfig.java | 621 ++++++-
.../org/apache/rocketmq/common/BrokerIdentity.java | 149 ++
.../org/apache/rocketmq/common/BrokerSyncInfo.java | 70 +
.../org/apache/rocketmq/common/ConfigManager.java | 11 +
.../org/apache/rocketmq/common/Configuration.java | 41 +
.../org/apache/rocketmq/common/DataVersion.java | 49 +-
.../{ThreadFactoryImpl.java => KeyBuilder.java} | 32 +-
.../org/apache/rocketmq/common/LockCallback.java | 12 +-
.../java/org/apache/rocketmq/common/MQVersion.java | 2 +-
.../java/org/apache/rocketmq/common/MixAll.java | 25 +-
.../apache/rocketmq/common/PopAckConstants.java | 44 +
.../org/apache/rocketmq/common/ServiceThread.java | 8 +-
.../apache/rocketmq/common/ThreadFactoryImpl.java | 13 +
.../apache/rocketmq/common/TopicAttributes.java | 47 +
.../org/apache/rocketmq/common/TopicConfig.java | 122 +-
.../org/apache/rocketmq/common/TopicQueueId.java | 54 +
.../MessageType.java => UnlockCallback.java} | 10 +-
.../java/org/apache/rocketmq/common/UtilAll.java | 129 +-
.../apache/rocketmq/common/admin/TopicOffset.java | 9 +
.../rocketmq/common/attribute/Attribute.java | 36 +-
.../rocketmq/common/attribute/AttributeParser.java | 79 +
.../BooleanAttribute.java} | 28 +-
.../MessageType.java => attribute/CQType.java} | 10 +-
.../EnumAttribute.java} | 30 +-
.../LongRangeAttribute.java} | 33 +-
.../TopicMessageType.java} | 29 +-
.../ConsumeInitMode.java} | 11 +-
.../rocketmq/common/constant/LoggerName.java | 9 +-
.../apache/rocketmq/common/constant/PermName.java | 18 +-
.../fastjson/GenericMapSuperclassDeserializer.java | 58 +
.../rocketmq/common/future/FutureTaskExt.java | 31 +-
.../rocketmq/common/message/MessageConst.java | 34 +-
.../rocketmq/common/message/MessageDecoder.java | 201 ++-
.../apache/rocketmq/common/message/MessageExt.java | 55 +
.../rocketmq/common/message/MessageExtBatch.java | 15 +-
.../common/message}/MessageExtBrokerInner.java | 3 +-
.../rocketmq/common/message/MessageQueue.java | 6 +
.../common/message/MessageQueueAssignment.java | 83 +
.../{MessageType.java => MessageRequestMode.java} | 30 +-
.../rocketmq/common/message/MessageType.java | 28 +-
.../rocketmq/common/message/MessageVersion.java | 70 +
.../common/namesrv/DefaultTopAddressing.java | 165 ++
.../NameServerUpdateCallback.java} | 10 +-
.../rocketmq/common/namesrv/NamesrvConfig.java | 134 +-
.../rocketmq/common/namesrv/TopAddressing.java | 88 +-
.../rocketmq/common/protocol/ForbiddenType.java | 38 +-
.../rocketmq/common/protocol/RequestCode.java | 42 +
.../rocketmq/common/protocol/ResponseCode.java | 18 +
.../common/protocol/body/BrokerMemberGroup.java | 90 +
.../protocol/body/CheckClientRequestBody.java | 9 +
.../rocketmq/common/protocol/body/ClusterInfo.java | 28 +-
.../body/ConsumerOffsetSerializeWrapper.java | 10 +
.../common/protocol/body/ConsumerRunningInfo.java | 63 +-
.../body/GetBrokerMemberGroupResponseBody.java | 19 +-
.../protocol/body/GetRemoteClientConfigBody.java | 26 +-
.../common/protocol/body/HARuntimeInfo.java | 188 ++
.../common/protocol/body/LockBatchRequestBody.java | 9 +
...ava => MessageRequestModeSerializeWrapper.java} | 20 +-
...ntRequestBody.java => PopProcessQueueInfo.java} | 45 +-
...stBody.java => QueryAssignmentRequestBody.java} | 39 +-
.../protocol/body/QueryAssignmentResponseBody.java | 22 +-
...ody.java => QuerySubscriptionResponseBody.java} | 22 +-
.../common/protocol/body/RegisterBrokerBody.java | 32 +-
.../SetMessageRequestModeRequestBody.java} | 54 +-
.../TopicConfigAndMappingSerializeWrapper.java | 68 +
...java => TopicQueueMappingSerializeWrapper.java} | 35 +-
.../protocol/body/UnlockBatchRequestBody.java | 9 +
...estHeader.java => AckMessageRequestHeader.java} | 37 +-
...uestHeader.java => AddBrokerRequestHeader.java} | 21 +-
....java => ChangeInvisibleTimeRequestHeader.java} | 51 +-
...java => ChangeInvisibleTimeResponseHeader.java} | 39 +-
.../ConsumeMessageDirectlyResultRequestHeader.java | 30 +
.../protocol/header/CreateTopicRequestHeader.java | 21 +
.../DeleteSubscriptionGroupRequestHeader.java | 10 +-
...eader.java => ExchangeHAInfoRequestHeader.java} | 44 +-
...ader.java => ExchangeHAInfoResponseHeader.java} | 44 +-
.../common/protocol/header/ExtraInfoUtil.java | 258 +++
...java => GetBrokerMemberGroupRequestHeader.java} | 31 +-
.../GetEarliestMsgStoretimeRequestHeader.java | 8 +-
.../protocol/header/GetMaxOffsetRequestHeader.java | 26 +-
.../protocol/header/GetMinOffsetRequestHeader.java | 8 +-
...> GetSubscriptionGroupConfigRequestHeader.java} | 24 +-
...eader.java => GetTopicConfigRequestHeader.java} | 19 +-
.../header/GetTopicStatsInfoRequestHeader.java | 4 +-
...r.java => InitConsumerOffsetRequestHeader.java} | 17 +-
...tHeader.java => NotificationRequestHeader.java} | 44 +-
...Header.java => NotificationResponseHeader.java} | 19 +-
...a => NotifyMinBrokerIdChangeRequestHeader.java} | 52 +-
...stHeader.java => PeekMessageRequestHeader.java} | 28 +-
...stHeader.java => PollingInfoRequestHeader.java} | 16 +-
...tHeader.java => PollingInfoResponseHeader.java} | 18 +-
.../protocol/header/PopMessageRequestHeader.java | 155 ++
.../protocol/header/PopMessageResponseHeader.java | 102 ++
.../protocol/header/PullMessageRequestHeader.java | 56 +-
.../protocol/header/PullMessageResponseHeader.java | 66 +
.../header/QueryConsumerOffsetRequestHeader.java | 18 +-
... QuerySubscriptionByConsumerRequestHeader.java} | 24 +-
...ava => QueryTopicsByConsumerRequestHeader.java} | 16 +-
...tHeader.java => RemoveBrokerRequestHeader.java} | 39 +-
...ader.java => ResetMasterFlushOffsetHeader.java} | 13 +-
.../protocol/header/SearchOffsetRequestHeader.java | 8 +-
.../protocol/header/SendMessageRequestHeader.java | 8 +-
.../protocol/header/SendMessageResponseHeader.java | 15 +
...r.java => StatisticsMessagesRequestHeader.java} | 37 +-
.../header/UpdateConsumerOffsetRequestHeader.java | 8 +-
...java => UpdateGroupForbiddenRequestHeader.java} | 28 +-
...ader.java => BrokerHeartbeatRequestHeader.java} | 58 +-
.../header/namesrv/GetRouteInfoRequestHeader.java | 2 +
.../namesrv/RegisterBrokerRequestHeader.java | 22 +
.../RegisterTopicRequestHeader.java} | 5 +-
.../rocketmq/common/protocol/route/BrokerData.java | 55 +-
.../route/MessageQueueRouteState.java} | 15 +-
.../rocketmq/common/protocol/route/QueueData.java | 13 +
.../common/protocol/route/TopicRouteData.java | 92 +-
.../common/protocol/route/TopicRouteDatas.java | 21 +-
.../apache/rocketmq/common/rpc/ClientMetadata.java | 176 ++
.../apache/rocketmq/common/rpc/RequestBuilder.java | 81 +
.../org/apache/rocketmq/common/rpc/RpcClient.java | 29 +-
.../apache/rocketmq/common/rpc/RpcClientHook.java | 12 +-
.../apache/rocketmq/common/rpc/RpcClientImpl.java | 340 ++++
.../apache/rocketmq/common/rpc/RpcClientUtils.java | 58 +
.../apache/rocketmq/common/rpc/RpcException.java | 31 +-
.../org/apache/rocketmq/common/rpc/RpcRequest.java | 33 +-
.../RpcRequestHeader.java} | 54 +-
.../apache/rocketmq/common/rpc/RpcResponse.java | 70 +
.../common/rpc/TopicQueueRequestHeader.java | 10 +-
.../rocketmq/common/rpc/TopicRequestHeader.java | 19 +-
.../common/statictopic/LogicQueueMappingItem.java | 212 +++
.../statictopic/TopicConfigAndQueueMapping.java | 63 +
.../statictopic/TopicQueueMappingContext.java | 99 ++
.../statictopic/TopicQueueMappingDetail.java | 144 ++
.../common/statictopic/TopicQueueMappingInfo.java | 161 ++
.../common/statictopic/TopicQueueMappingOne.java | 85 +
.../common/statictopic/TopicQueueMappingUtils.java | 694 ++++++++
.../statictopic/TopicRemappingDetailWrapper.java | 104 ++
.../rocketmq/common/statistics/FutureHolder.java | 53 +
.../Interceptor.java} | 18 +-
.../common/statistics/StatisticsBrief.java | 184 ++
.../statistics/StatisticsBriefInterceptor.java | 76 +
.../rocketmq/common/statistics/StatisticsItem.java | 175 ++
.../StatisticsItemFormatter.java} | 32 +-
.../common/statistics/StatisticsItemPrinter.java | 50 +
.../StatisticsItemScheduledIncrementPrinter.java | 290 +++
.../statistics/StatisticsItemScheduledPrinter.java | 97 ++
.../StatisticsItemStateGetter.java} | 10 +-
.../StatisticsKindMeta.java} | 39 +-
.../common/statistics/StatisticsManager.java | 157 ++
.../common/subscription/CustomizedRetryPolicy.java | 85 +
.../subscription/ExponentialRetryPolicy.java | 74 +
.../common/subscription/GroupForbidden.java | 86 +
.../common/subscription/GroupRetryPolicy.java | 78 +
.../GroupRetryPolicyType.java} | 12 +-
.../rocketmq/common/subscription/RetryPolicy.java | 17 +-
.../subscription/SubscriptionGroupConfig.java | 95 +-
.../rocketmq/common/sysflag/MessageSysFlag.java | 5 +-
.../rocketmq/common/sysflag/PullSysFlag.java | 4 +
.../rocketmq/common/topic/TopicValidator.java | 2 +
.../rocketmq/common/utils/DataConverter.java | 35 +-
.../apache/rocketmq/common/utils/MessageUtils.java | 49 +
.../common/utils/PositiveAtomicCounter.java | 32 +-
.../rocketmq/common/utils/QueueTypeUtils.java | 51 +
.../rocketmq/common/utils}/ServiceProvider.java | 27 +-
.../apache/rocketmq/common/utils/ThreadUtils.java | 17 +-
.../apache/rocketmq/common/ConfigManagerTest.java | 5 +-
.../apache/rocketmq/common/DataVersionTest.java | 7 +
.../rocketmq/common/RegisterBrokerBodyTest.java | 5 +-
.../apache/rocketmq/common/TopicConfigTest.java | 78 +
.../org/apache/rocketmq/common/UtilAllTest.java | 21 +-
.../common/attribute/AttributeParserTest.java | 70 +
.../rocketmq/common/attribute/AttributeTest.java | 70 +
.../common/message/MessageClientIDSetterTest.java | 2 -
.../common/message/MessageDecoderTest.java | 12 +-
.../rocketmq/common/message/MessageTest.java | 1 -
.../rocketmq/common/protocol/body/KVTableTest.java | 1 -
.../MessageRequestModeSerializeWrapperTest.java | 58 +
.../common/protocol/route/TopicRouteDataTest.java | 11 +-
.../common/statictopic/TopicQueueMappingTest.java | 78 +
.../statictopic/TopicQueueMappingUtilsTest.java | 320 ++++
.../subscription/CustomizedRetryPolicyTest.java | 44 +
.../subscription/ExponentialRetryPolicyTest.java | 44 +
.../common/subscription/GroupRetryPolicyTest.java | 49 +
{logging => container}/pom.xml | 23 +-
.../apache/rocketmq/container/BrokerBootHook.java | 36 +-
.../apache/rocketmq/container/BrokerContainer.java | 478 +++++
.../rocketmq/container/BrokerContainerConfig.java | 75 +
.../container/BrokerContainerProcessor.java | 280 +++
.../rocketmq/container/BrokerContainerStartup.java | 445 +++++
.../ContainerClientHouseKeepingService.java | 104 ++
.../rocketmq/container/IBrokerContainer.java | 142 ++
.../rocketmq/container/InnerBrokerController.java | 204 +++
.../container/InnerSalveBrokerController.java | 46 +
.../logback/BrokerLogbackConfigurator.java | 187 ++
.../container/BrokerContainerStartupTest.java | 140 ++
.../rocketmq/container/BrokerContainerTest.java | 371 ++++
.../rocketmq/container/BrokerPreOnlineTest.java | 102 ++
distribution/bin/{mqshutdown => mqbrokercontainer} | 46 +-
distribution/bin/mqshutdown | 14 +
distribution/bin/runbroker.sh | 2 +-
.../2container-2m-2s/broker-a-in-container1.conf | 49 +-
.../2container-2m-2s/broker-a-in-container2.conf | 49 +-
.../2container-2m-2s/broker-b-in-container1.conf | 49 +-
.../2container-2m-2s/broker-b-in-container2.conf | 49 +-
.../2container-2m-2s/broker-container1.conf | 38 +-
.../2container-2m-2s/broker-container2.conf | 38 +-
.../container/2container-2m-2s/nameserver.conf | 30 +-
.../conf/container/broker-a.conf | 48 +-
.../conf/container/broker-b.conf | 48 +-
.../conf/container/broker-container.conf | 38 +-
distribution/conf/logback_broker.xml | 29 +
distribution/conf/logback_tools.xml | 5 +
distribution/pom.xml | 6 +-
distribution/release.xml | 1 +
docs/cn/BrokerContainer.md | 152 ++
docs/cn/QuorumACK.md | 70 +
docs/cn/README.md | 12 +-
docs/cn/SlaveActingMasterMode.md | 164 ++
..._Topic_Logic_Queue_\350\256\276\350\256\241.md" | 503 ++++++
docs/cn/statictopic/The_Scope_Of_Static_Topic.md | 116 ++
docs/en/Feature.md | 7 +-
example/pom.xml | 6 +-
.../rocketmq/example/benchmark/Consumer.java | 2 +
.../rocketmq/example/simple/PopPushConsumer.java | 62 +
filter/pom.xml | 2 +-
logging/pom.xml | 2 +-
.../rocketmq/logging/InternalLoggerFactory.java | 11 +
.../rocketmq/logging/Slf4jLoggerFactory.java | 87 +-
.../apache/rocketmq/logging/inner/SysLogger.java | 4 +-
.../rocketmq/logging/Slf4jLoggerFactoryTest.java | 2 +-
.../rocketmq/logging/inner/LoggingBuilderTest.java | 8 +-
namesrv/pom.xml | 14 +-
.../apache/rocketmq/namesrv/NamesrvController.java | 149 +-
.../apache/rocketmq/namesrv/NamesrvStartup.java | 11 +-
.../namesrv/processor/ClientRequestProcessor.java | 102 ++
.../processor/ClusterTestRequestProcessor.java | 2 +-
.../namesrv/processor/DefaultRequestProcessor.java | 364 ++--
.../namesrv/routeinfo/BatchUnRegisterService.java | 85 +
.../routeinfo/BrokerHousekeepingService.java | 6 +-
.../namesrv/routeinfo/RouteInfoManager.java | 1027 ++++++++---
...rocessorTest.java => RequestProcessorTest.java} | 159 +-
.../namesrv/routeinfo/GetRouteInfoBenchmark.java | 148 ++
.../namesrv/routeinfo/RegisterBrokerBenchmark.java | 177 ++
.../routeinfo/RouteInfoManagerBrokerPermTest.java | 24 +-
.../RouteInfoManagerBrokerRegisterTest.java | 61 +-
.../RouteInfoManagerStaticRegisterTest.java | 19 +-
.../namesrv/routeinfo/RouteInfoManagerTest.java | 223 +++
.../routeinfo/RouteInfoManagerTestBase.java | 1 +
.../routeinfo/RouteInfoManager_NewTest.java | 783 +++++++++
openmessaging/pom.xml | 2 +-
pom.xml | 22 +-
remoting/pom.xml | 2 +-
.../java/org/apache/rocketmq/remoting/RPCHook.java | 2 +-
.../apache/rocketmq/remoting/RemotingClient.java | 13 +-
.../apache/rocketmq/remoting/RemotingServer.java | 6 +
.../apache/rocketmq/remoting/RemotingService.java | 5 +
.../rocketmq/remoting/common/RemotingHelper.java | 39 +
.../rocketmq/remoting/common/RemotingUtil.java | 18 +-
.../rocketmq/remoting/netty/NettyClientConfig.java | 38 +
.../remoting/netty/NettyRemotingAbstract.java | 103 +-
.../remoting/netty/NettyRemotingClient.java | 392 ++++-
.../remoting/netty/NettyRemotingServer.java | 158 +-
.../remoting/netty/NettyRequestProcessor.java | 1 -
.../rocketmq/remoting/netty/ResponseFuture.java | 42 +-
.../remoting/protocol/FastCodesHeader.java | 2 +-
.../remoting/protocol/RemotingCommand.java | 54 +-
.../remoting/protocol/RemotingSerializable.java | 7 +-
.../rocketmq/remoting/RemotingServerTest.java | 5 +-
.../rocketmq/remoting/SubRemotingServerTest.java | 109 ++
.../remoting/netty/NettyRemotingAbstractTest.java | 6 +-
.../remoting/protocol/RemotingCommandTest.java | 99 +-
srvutil/pom.xml | 10 +-
.../rocketmq/srvutil/ConcurrentHashMapUtil.java | 50 +
.../apache/rocketmq/srvutil/FileWatchService.java | 4 +-
.../rocketmq/util/cache/CacheEvictHandler.java | 10 +-
.../apache/rocketmq/util/cache/CacheObject.java | 28 +-
.../rocketmq/util/cache/ExpiredLocalCache.java | 84 +
.../org/apache/rocketmq/util/cache/LocalCache.java | 58 +
.../apache/rocketmq/util/cache/LockManager.java | 54 +
store/pom.xml | 2 +-
.../rocketmq/store/AllocateMappedFileService.java | 11 +-
.../rocketmq/store/AppendMessageCallback.java | 2 +-
.../apache/rocketmq/store/AppendMessageResult.java | 12 +
.../java/org/apache/rocketmq/store/CommitLog.java | 1138 ++++++++----
.../org/apache/rocketmq/store/ConsumeQueue.java | 313 +++-
.../org/apache/rocketmq/store/ConsumeQueueExt.java | 5 +
.../apache/rocketmq/store/DefaultMessageStore.java | 1842 ++++++++++++--------
.../org/apache/rocketmq/store/DispatchRequest.java | 45 +
.../apache/rocketmq/store/FileQueueSnapshot.java | 90 +
.../apache/rocketmq/store/GetMessageResult.java | 37 +-
.../org/apache/rocketmq/store/MappedFileQueue.java | 189 +-
.../org/apache/rocketmq/store/MessageStore.java | 422 ++++-
.../org/apache/rocketmq/store/MultiDispatch.java | 184 --
.../rocketmq/store/MultiPathMappedFileQueue.java | 6 +-
.../apache/rocketmq/store/PutMessageContext.java | 48 +
.../apache/rocketmq/store/PutMessageResult.java | 26 +-
.../apache/rocketmq/store/PutMessageStatus.java | 6 +-
.../apache/rocketmq/store/QueryMessageResult.java | 4 +
.../rocketmq/store/SelectMappedBufferResult.java | 11 +-
.../rocketmq/store/SelectMappedFileResult.java | 34 +-
.../org/apache/rocketmq/store/StoreCheckpoint.java | 21 +-
.../apache/rocketmq/store/StoreStatsService.java | 11 +
.../java/org/apache/rocketmq/store/StoreUtil.java | 44 +
.../java/org/apache/rocketmq/store/Swappable.java | 14 +-
.../org/apache/rocketmq/store/TopicQueueLock.java | 46 +
.../rocketmq/store/config/MessageStoreConfig.java | 484 ++++-
.../store/config/StorePathConfigHelper.java | 3 +
.../rocketmq/store/dledger/DLedgerCommitLog.java | 275 ++-
.../apache/rocketmq/store/ha/DefaultHAClient.java | 392 +++++
...{HAConnection.java => DefaultHAConnection.java} | 187 +-
.../apache/rocketmq/store/ha/DefaultHAService.java | 339 ++++
.../org/apache/rocketmq/store/ha/FlowMonitor.java | 76 +
.../rocketmq/store/ha/GroupTransferService.java | 140 ++
.../org/apache/rocketmq/store/ha/HAClient.java | 104 ++
.../org/apache/rocketmq/store/ha/HAConnection.java | 428 +----
.../rocketmq/store/ha/HAConnectionState.java | 31 +-
.../ha/HAConnectionStateNotificationRequest.java | 49 +
.../ha/HAConnectionStateNotificationService.java | 150 ++
.../org/apache/rocketmq/store/ha/HAService.java | 665 +------
.../apache/rocketmq/store/ha/WaitNotifyObject.java | 67 +-
.../apache/rocketmq/store/hook/PutMessageHook.java | 25 +-
.../rocketmq/store/hook/SendMessageBackHook.java | 25 +-
.../org/apache/rocketmq/store/index/IndexFile.java | 21 +-
.../apache/rocketmq/store/index/IndexService.java | 75 +-
.../rocketmq/store/logfile/AbstractMappedFile.java | 9 +-
.../DefaultMappedFile.java} | 302 ++--
.../apache/rocketmq/store/logfile/MappedFile.java | 340 ++++
.../java/org/apache/rocketmq/store/pop/AckMsg.java | 97 ++
.../apache/rocketmq/store/pop/PopCheckPoint.java | 184 ++
.../rocketmq/store/queue/BatchConsumeQueue.java | 980 +++++++++++
.../rocketmq/store/queue/BatchOffsetIndex.java | 57 +
.../store/queue/ConsumeQueueInterface.java | 142 ++
.../rocketmq/store/queue/ConsumeQueueStore.java | 483 +++++
.../org/apache/rocketmq/store/queue/CqUnit.java | 115 ++
.../rocketmq/store/queue/FileQueueLifeCycle.java | 84 +
.../rocketmq/store/queue/QueueOffsetAssigner.java | 92 +
.../rocketmq/store/queue/ReferredIterator.java | 14 +-
.../apache/rocketmq/store/stats/BrokerStats.java | 6 +-
.../rocketmq/store/stats/BrokerStatsManager.java | 435 ++++-
.../apache/rocketmq/store/util/PerfCounter.java | 370 ++++
.../apache/rocketmq/store/AppendCallbackTest.java | 4 +-
.../apache/rocketmq/store/BatchPutMessageTest.java | 61 +-
.../apache/rocketmq/store/ConsumeQueueTest.java | 80 +-
.../store/DefaultMessageStoreCleanFilesTest.java | 92 +-
.../store/DefaultMessageStoreShutDownTest.java | 2 +-
.../rocketmq/store/DefaultMessageStoreTest.java | 285 ++-
.../java/org/apache/rocketmq/store/HATest.java | 130 +-
.../apache/rocketmq/store/MappedFileQueueTest.java | 156 +-
.../org/apache/rocketmq/store/MappedFileTest.java | 4 +-
.../apache/rocketmq/store/MultiDispatchTest.java | 63 +-
.../store/MultiPathMappedFileQueueTest.java | 32 +-
.../rocketmq/store/ScheduleMessageServiceTest.java | 194 ---
.../org/apache/rocketmq/store/StoreTestBase.java | 13 +-
.../store/dledger/DLedgerCommitlogTest.java | 20 +-
.../store/dledger/MessageStoreTestBase.java | 5 +-
.../rocketmq/store/dledger/MixCommitlogTest.java | 2 -
.../apache/rocketmq/store/ha/FlowMonitorTest.java | 61 +
.../org/apache/rocketmq/store/ha/HAClientTest.java | 72 +
.../org/apache/rocketmq/store/ha/HAServerTest.java | 295 ++++
.../rocketmq/store/ha/WaitNotifyObjectTest.java | 2 -
.../store/queue/BatchConsumeMessageTest.java | 454 +++++
.../store/queue/BatchConsumeQueueTest.java | 312 ++++
.../store/queue/ConsumeQueueStoreTest.java | 100 ++
.../rocketmq/store/queue/ConsumeQueueTest.java | 102 ++
.../apache/rocketmq/store/queue/QueueTestBase.java | 114 ++
test/pom.xml | 6 +-
.../test/client/rmq/RMQNormalConsumer.java | 5 +
.../test/client/rmq/RMQNormalProducer.java | 7 +
.../rocketmq/test/client/rmq/RMQPopConsumer.java | 33 +
.../test/clientinterface/AbstractMQProducer.java | 1 +
.../rocketmq/test/factory/ConsumerFactory.java | 10 +
.../rocketmq/test/listener/AbstractListener.java | 2 +-
.../org/apache/rocketmq/test/util/MQAdmin.java | 166 --
.../rocketmq/test/util/MQAdminTestUtils.java | 310 ++++
.../org/apache/rocketmq/test/base/BaseConf.java | 168 +-
.../rocketmq/test/base/IntegrationTestBase.java | 40 +-
.../base/dledger/DLedgerProduceAndConsumeIT.java | 3 +-
.../consumer/balance/NormalMsgStaticBalanceIT.java | 18 +-
.../normal/BroadCastNormalMsgRecvFailIT.java | 2 +
.../test/client/consumer/pop/PopSubCheckIT.java | 96 +
.../test/client/producer/batch/BatchSendIT.java | 161 ++
.../client/producer/oneway/OneWaySendWithMQIT.java | 10 -
.../client/producer/order/OrderMsgRebalanceIT.java | 2 +-
.../test/container/AddAndRemoveBrokerIT.java | 83 +
.../rocketmq/test/container/BrokerFailoverIT.java | 86 +
.../test/container/BrokerMemberGroupIT.java | 71 +
.../container/ContainerIntegrationTestBase.java | 666 +++++++
.../test/container/GetMaxOffsetFromSlaveIT.java | 100 ++
.../test/container/GetMetadataReverseIT.java | 231 +++
.../test/container/PullMultipleReplicasIT.java | 201 +++
.../test/container/PushMultipleReplicasIT.java | 114 ++
.../test/container/RebalanceLockOnSlaveIT.java | 209 +++
.../container/ScheduleSlaveActingMasterIT.java | 220 +++
.../test/container/ScheduledMessageIT.java | 153 ++
.../test/container/SendMultipleReplicasIT.java | 159 ++
.../rocketmq/test/container/SlaveBrokerIT.java | 117 ++
.../test/container/SyncConsumerOffsetIT.java | 148 ++
.../rocketmq/test/delay/NormalMsgDelayIT.java | 1 -
.../rocketmq/test/offset/OffsetNotFoundIT.java | 132 ++
.../test/smoke/NormalMessageSendAndRecvIT.java | 6 +
.../rocketmq/test/statictopic/StaticTopicIT.java | 521 ++++++
tools/pom.xml | 6 +-
.../rocketmq/tools/admin/DefaultMQAdminExt.java | 208 ++-
.../tools/admin/DefaultMQAdminExtImpl.java | 1149 ++++++++----
.../apache/rocketmq/tools/admin/MQAdminExt.java | 128 +-
.../apache/rocketmq/tools/admin/MQAdminUtils.java | 343 ++++
.../tools/admin/api/BrokerOperatorResult.java | 37 +-
.../tools/admin/common/AdminToolHandler.java | 10 +-
.../tools/admin/common/AdminToolResult.java | 76 +
.../common/AdminToolsResultCodeEnum.java} | 31 +-
.../apache/rocketmq/tools/command/CommandUtil.java | 17 +-
.../rocketmq/tools/command/MQAdminStartup.java | 24 +-
.../tools/command/SubCommandException.java | 4 +
.../command/broker/GetBrokerConfigCommand.java | 29 +-
.../broker/ResetMasterFlushOffsetSubCommand.java | 71 +
.../broker/UpdateBrokerConfigSubCommand.java | 16 +-
.../command/cluster/CLusterSendMsgRTCommand.java | 4 +-
.../command/cluster/ClusterListSubCommand.java | 106 +-
.../consumer/ConsumerProgressSubCommand.java | 17 +-
.../consumer/DeleteSubscriptionGroupCommand.java | 19 +-
.../consumer/GetConsumerConfigSubCommand.java | 2 +-
.../SetConsumeModeSubCommand.java} | 71 +-
.../command/consumer/UpdateSubGroupSubCommand.java | 26 +
.../command/container/AddBrokerSubCommand.java | 66 +
.../command/container/RemoveBrokerSubCommand.java | 79 +
.../tools/command/ha/HAStatusSubCommand.java | 151 ++
.../command/offset/ResetOffsetByTimeCommand.java | 34 +
.../tools/command/topic/DeleteTopicSubCommand.java | 2 +-
.../topic/RemappingStaticTopicSubCommand.java | 207 +++
.../command/topic/UpdateStaticTopicSubCommand.java | 208 +++
.../tools/command/topic/UpdateTopicSubCommand.java | 14 +
.../tools/admin/DefaultMQAdminExtTest.java | 74 +-
.../rocketmq/tools/command/CommandUtilTest.java | 2 +-
.../consumer/ConsumerProgressSubCommandTest.java | 2 +
.../message/QueryMsgByUniqueKeySubCommandTest.java | 18 +-
602 files changed, 52645 insertions(+), 9325 deletions(-)
diff --cc broker/pom.xml
index de2063114,8e52c1904..14a0b70f9
--- a/broker/pom.xml
+++ b/broker/pom.xml
@@@ -46,10 -46,10 +46,14 @@@
<groupId>${project.groupId}</groupId>
<artifactId>rocketmq-filter</artifactId>
</dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-acl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
@@@ -70,6 -74,14 +74,10 @@@
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.googlecode.concurrentlinkedhashmap</groupId>
+ <artifactId>concurrentlinkedhashmap-lru</artifactId>
+ </dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-acl</artifactId>
- </dependency>
</dependencies>
<build>
diff --cc broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
index c5b53a777,78e1dec27..9bfcc0f21
--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
@@@ -57,21 -93,36 +93,36 @@@ import org.apache.rocketmq.remoting.net
import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+ import java.io.UnsupportedEncodingException;
+ import java.util.Arrays;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Set;
+ import java.util.concurrent.ArrayBlockingQueue;
+ import java.util.concurrent.CopyOnWriteArrayList;
+ import java.util.concurrent.CountDownLatch;
+ import java.util.concurrent.TimeUnit;
+
public class BrokerOuterAPI {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final RemotingClient remotingClient;
- private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr());
+ private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr());
private String nameSrvAddr = null;
private BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES,
- new ArrayBlockingQueue<Runnable>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true));
+ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true));
+ private ClientMetadata clientMetadata;
+ private RpcClient rpcClient;
+
public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
- this(nettyClientConfig, null);
+ this(nettyClientConfig, null, new ClientMetadata());
}
- public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook) {
+ private BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook, ClientMetadata clientMetadata) {
this.remotingClient = new NettyRemotingClient(nettyClientConfig);
+ this.clientMetadata = clientMetadata;
this.remotingClient.registerRPCHook(rpcHook);
+ this.rpcClient = new RpcClientImpl(this.clientMetadata, this.remotingClient);
}
public void start() {
@@@ -101,13 -156,202 +156,202 @@@
}
public void updateNameServerAddressList(final String addrs) {
- List<String> lst = new ArrayList<String>();
String[] addrArray = addrs.split(";");
- for (String addr : addrArray) {
- lst.add(addr);
+ List<String> lst = new ArrayList<String>(Arrays.asList(addrArray));
+ this.remotingClient.updateNameServerAddressList(lst);
+ }
+
+ public BrokerMemberGroup syncBrokerMemberGroup(String clusterName, String brokerName)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ return syncBrokerMemberGroup(clusterName, brokerName, false);
+ }
+
+ public BrokerMemberGroup syncBrokerMemberGroup(String clusterName, String brokerName,
+ boolean isCompatibleWithOldNameSrv)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ if (isCompatibleWithOldNameSrv) {
+ return getBrokerMemberGroupCompatible(clusterName, brokerName);
+ } else {
+ return getBrokerMemberGroup(clusterName, brokerName);
}
+ }
- this.remotingClient.updateNameServerAddressList(lst);
+ public BrokerMemberGroup getBrokerMemberGroup(String clusterName, String brokerName)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ BrokerMemberGroup brokerMemberGroup = new BrokerMemberGroup(clusterName, brokerName);
+
+ GetBrokerMemberGroupRequestHeader requestHeader = new GetBrokerMemberGroupRequestHeader();
+ requestHeader.setClusterName(clusterName);
+ requestHeader.setBrokerName(brokerName);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_MEMBER_GROUP, requestHeader);
+
+ RemotingCommand response = null;
+ response = this.remotingClient.invokeSync(null, request, 3000);
+ assert response != null;
+
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ byte[] body = response.getBody();
+ if (body != null) {
+ GetBrokerMemberGroupResponseBody brokerMemberGroupResponseBody =
+ GetBrokerMemberGroupResponseBody.decode(body, GetBrokerMemberGroupResponseBody.class);
+
+ return brokerMemberGroupResponseBody.getBrokerMemberGroup();
+ }
+ }
+ default:
+ break;
+ }
+
+ return brokerMemberGroup;
+ }
+
+ public BrokerMemberGroup getBrokerMemberGroupCompatible(String clusterName, String brokerName)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ BrokerMemberGroup brokerMemberGroup = new BrokerMemberGroup(clusterName, brokerName);
+
+ GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();
+ requestHeader.setTopic(TopicValidator.SYNC_BROKER_MEMBER_GROUP_PREFIX + brokerName);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);
+
+ RemotingCommand response;
+ response = this.remotingClient.invokeSync(null, request, 3000);
+ assert response != null;
+
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ byte[] body = response.getBody();
+ if (body != null) {
+ TopicRouteData topicRouteData = TopicRouteData.decode(body, TopicRouteData.class);
+ for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
+ if (brokerData != null
+ && brokerData.getBrokerName().equals(brokerName)
+ && brokerData.getCluster().equals(clusterName)) {
+ brokerMemberGroup.getBrokerAddrs().putAll(brokerData.getBrokerAddrs());
+ break;
+ }
+ }
+ return brokerMemberGroup;
+ }
+ }
+ default:
+ break;
+ }
+
+ return brokerMemberGroup;
+ }
+
+ public void sendHeartbeatViaDataVersion(
+ final String clusterName,
+ final String brokerAddr,
+ final String brokerName,
+ final Long brokerId,
+ final int timeoutMillis,
+ final DataVersion dataVersion,
+ final boolean isInBrokerContainer) {
+ List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
+ if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
+ final QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
+ requestHeader.setBrokerAddr(brokerAddr);
+ requestHeader.setBrokerName(brokerName);
+ requestHeader.setBrokerId(brokerId);
+ requestHeader.setClusterName(clusterName);
+
+ for (final String namesrvAddr : nameServerAddressList) {
+ brokerOuterExecutor.execute(new AbstractBrokerRunnable(new BrokerIdentity(clusterName, brokerName, brokerId, isInBrokerContainer)) {
+
+ @Override
+ public void run2() {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);
+ request.setBody(dataVersion.encode());
+
+ try {
+ BrokerOuterAPI.this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMillis);
+ } catch (Exception e) {
+ LOGGER.error("sendHeartbeat Exception " + namesrvAddr, e);
+ }
+ }
+ });
+ }
+ }
+ }
+
+ public void sendHeartbeat(final String clusterName,
+ final String brokerAddr,
+ final String brokerName,
+ final Long brokerId,
+ final int timeoutMills,
+ final boolean isInBrokerContainer) {
+ List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
+
+ final BrokerHeartbeatRequestHeader requestHeader = new BrokerHeartbeatRequestHeader();
+ requestHeader.setClusterName(clusterName);
+ requestHeader.setBrokerAddr(brokerAddr);
+ requestHeader.setBrokerName(brokerName);
+
+ if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
+ for (final String namesrvAddr : nameServerAddressList) {
+ brokerOuterExecutor.execute(new AbstractBrokerRunnable(new BrokerIdentity(clusterName, brokerName, brokerId, isInBrokerContainer)) {
+ @Override
+ public void run2() {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BROKER_HEARTBEAT, requestHeader);
+
+ try {
+ BrokerOuterAPI.this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMills);
+ } catch (Exception e) {
+ LOGGER.error("sendHeartbeat Exception " + namesrvAddr, e);
+ }
+ }
+ });
+ }
+ }
+ }
+
+ public BrokerSyncInfo retrieveBrokerHaInfo(String masterBrokerAddr)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,
+ MQBrokerException, RemotingCommandException {
+ ExchangeHAInfoRequestHeader requestHeader = new ExchangeHAInfoRequestHeader();
+ requestHeader.setMasterHaAddress(null);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(masterBrokerAddr, request, 3000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
- ExchangeHAInfoResponseHeader responseHeader = response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
++ ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
+ return new BrokerSyncInfo(responseHeader.getMasterHaAddress(), responseHeader.getMasterFlushOffset(), responseHeader.getMasterAddress());
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public void sendBrokerHaInfo(String brokerAddr, String masterHaAddr, long brokerInitMaxOffset, String masterAddr)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
+ ExchangeHAInfoRequestHeader requestHeader = new ExchangeHAInfoRequestHeader();
+ requestHeader.setMasterHaAddress(masterHaAddr);
+ requestHeader.setMasterFlushOffset(brokerInitMaxOffset);
+ requestHeader.setMasterAddress(masterAddr);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000);
+
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ return;
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
}
public List<RegisterBrokerResult> registerBrokerAll(
@@@ -388,4 -689,303 +689,303 @@@
public void registerRPCHook(RPCHook rpcHook) {
remotingClient.registerRPCHook(rpcHook);
}
+
+ public void clearRPCHook() {
+ remotingClient.clearRPCHook();
+ }
+
+ public long getMaxOffset(final String addr, final String topic, final int queueId, final boolean committed,
+ final boolean isOnlyThisBroker)
+ throws RemotingException, MQBrokerException, InterruptedException {
+ GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();
+ requestHeader.setTopic(topic);
+ requestHeader.setQueueId(queueId);
+ requestHeader.setCommitted(committed);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
- GetMaxOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
++ GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
+
+ return responseHeader.getOffset();
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public long getMinOffset(final String addr, final String topic, final int queueId, final boolean isOnlyThisBroker)
+ throws RemotingException, MQBrokerException, InterruptedException {
+ GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader();
+ requestHeader.setTopic(topic);
+ requestHeader.setQueueId(queueId);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
- GetMinOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
++ GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
+
+ return responseHeader.getOffset();
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public void lockBatchMQAsync(
+ final String addr,
+ final LockBatchRequestBody requestBody,
+ final long timeoutMillis,
+ final LockCallback callback) throws RemotingException, InterruptedException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);
+
+ request.setBody(requestBody.encode());
+ this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
+ if (callback == null) {
+ return;
+ }
+
+ try {
+ RemotingCommand response = responseFuture.getResponseCommand();
+ if (response != null) {
+ if (response.getCode() == ResponseCode.SUCCESS) {
+ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(),
+ LockBatchResponseBody.class);
+ Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
+ callback.onSuccess(messageQueues);
+ } else {
+ callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
+ }
+ }
+ } catch (Throwable ignored) {
+
+ }
+ });
+ }
+
+ public void unlockBatchMQAsync(
+ final String addr,
+ final UnlockBatchRequestBody requestBody,
+ final long timeoutMillis,
+ final UnlockCallback callback) throws RemotingException, InterruptedException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, null);
+
+ request.setBody(requestBody.encode());
+
+ this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
+ if (callback == null) {
+ return;
+ }
+
+ try {
+ RemotingCommand response = responseFuture.getResponseCommand();
+ if (response != null) {
+ if (response.getCode() == ResponseCode.SUCCESS) {
+ callback.onSuccess();
+ } else {
+ callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
+ }
+ }
+ } catch (Throwable ignored) {
+
+ }
+ });
+ }
+
+ public RemotingClient getRemotingClient() {
+ return this.remotingClient;
+ }
+
+ public SendResult sendMessageToSpecificBroker(String brokerAddr, final String brokerName,
+ final MessageExt msg, String group,
+ long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException {
+
+ SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+ requestHeader.setProducerGroup(group);
+ requestHeader.setTopic(msg.getTopic());
+ requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);
+ requestHeader.setDefaultTopicQueueNums(8);
+ requestHeader.setQueueId(msg.getQueueId());
+ requestHeader.setSysFlag(msg.getSysFlag());
+ requestHeader.setBornTimestamp(msg.getBornTimestamp());
+ requestHeader.setFlag(msg.getFlag());
+ requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));
+ requestHeader.setReconsumeTimes(msg.getReconsumeTimes());
+ requestHeader.setBatch(false);
+
+ SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2);
+
+ request.setBody(msg.getBody());
+
+ RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
+
+ return this.processSendResponse(brokerName, msg, response);
+ }
+
+ private SendResult processSendResponse(
+ final String brokerName,
+ final Message msg,
+ final RemotingCommand response
+ ) throws MQBrokerException, RemotingCommandException {
+ switch (response.getCode()) {
+ case ResponseCode.FLUSH_DISK_TIMEOUT:
+ case ResponseCode.FLUSH_SLAVE_TIMEOUT:
+ case ResponseCode.SLAVE_NOT_AVAILABLE:
+ case ResponseCode.SUCCESS: {
+ SendStatus sendStatus = SendStatus.SEND_OK;
+ switch (response.getCode()) {
+ case ResponseCode.FLUSH_DISK_TIMEOUT:
+ sendStatus = SendStatus.FLUSH_DISK_TIMEOUT;
+ break;
+ case ResponseCode.FLUSH_SLAVE_TIMEOUT:
+ sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT;
+ break;
+ case ResponseCode.SLAVE_NOT_AVAILABLE:
+ sendStatus = SendStatus.SLAVE_NOT_AVAILABLE;
+ break;
+ case ResponseCode.SUCCESS:
+ sendStatus = SendStatus.SEND_OK;
+ break;
+ default:
+ assert false;
+ break;
+ }
+
+ SendMessageResponseHeader responseHeader =
+ (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
+
+ //If namespace not null , reset Topic without namespace.
+ String topic = msg.getTopic();
+
+ MessageQueue messageQueue = new MessageQueue(topic, brokerName, responseHeader.getQueueId());
+
+ String uniqMsgId = MessageClientIDSetter.getUniqID(msg);
+ if (msg instanceof MessageBatch) {
+ StringBuilder sb = new StringBuilder();
+ for (Message message : (MessageBatch) msg) {
+ sb.append(sb.length() == 0 ? "" : ",").append(MessageClientIDSetter.getUniqID(message));
+ }
+ uniqMsgId = sb.toString();
+ }
+ SendResult sendResult = new SendResult(sendStatus,
+ uniqMsgId,
+ responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());
+ sendResult.setTransactionId(responseHeader.getTransactionId());
+ String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);
+ String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH);
+ if (regionId == null || regionId.isEmpty()) {
+ regionId = MixAll.DEFAULT_TRACE_REGION_ID;
+ }
+ if (traceOn != null && traceOn.equals("false")) {
+ sendResult.setTraceOn(false);
+ } else {
+ sendResult.setTraceOn(true);
+ }
+ sendResult.setRegionId(regionId);
+ return sendResult;
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public BrokerFixedThreadPoolExecutor getBrokerOuterExecutor() {
+ return brokerOuterExecutor;
+ }
+
+ public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis)
+ throws RemotingException, MQBrokerException, InterruptedException {
+ return getTopicRouteInfoFromNameServer(topic, timeoutMillis, true);
+ }
+
+ public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis,
+ boolean allowTopicNotExist) throws MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();
+ requestHeader.setTopic(topic);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.TOPIC_NOT_EXIST: {
+ if (allowTopicNotExist) {
+ LOGGER.warn("get Topic [{}] RouteInfoFromNameServer is not exist value", topic);
+ }
+
+ break;
+ }
+ case ResponseCode.SUCCESS: {
+ byte[] body = response.getBody();
+ if (body != null) {
+ return TopicRouteData.decode(body, TopicRouteData.class);
+ }
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public ClusterInfo getBrokerClusterInfo() throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null);
+ RemotingCommand response = this.remotingClient.invokeSync(null, request, 3_000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ return ClusterInfo.decode(response.getBody(), ClusterInfo.class);
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public void forwardRequest(String brokerAddr, RemotingCommand request, long timeoutMillis,
+ InvokeCallback invokeCallback) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException, RemotingTooMuchRequestException, RemotingConnectException {
+ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, invokeCallback);
+ }
+
+ public void refreshMetadata() throws Exception {
+ ClusterInfo brokerClusterInfo = getBrokerClusterInfo();
+ clientMetadata.refreshClusterInfo(brokerClusterInfo);
+ }
+
+ public ClientMetadata getClientMetadata() {
+ return clientMetadata;
+ }
+
+ public RpcClient getRpcClient() {
+ return rpcClient;
+ }
+
+ public MessageRequestModeSerializeWrapper getAllMessageRequestMode(
+ final String addr) throws RemotingSendRequestException, RemotingConnectException,
+ MQBrokerException, RemotingTimeoutException, InterruptedException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_MESSAGE_REQUEST_MODE, null);
+ RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ return MessageRequestModeSerializeWrapper.decode(response.getBody(), MessageRequestModeSerializeWrapper.class);
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark(), addr);
+ }
}
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
index 21832e27a,18212397d..3d1267818
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
@@@ -220,10 -512,10 +512,10 @@@ public abstract class AbstractSendMessa
if (queueIdInt >= idValid) {
String errorInfo = String.format("request queueId[%d] is illegal, %s Producer: %s",
queueIdInt,
- topicConfig.toString(),
+ topicConfig,
RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
- log.warn(errorInfo);
+ LOGGER.warn(errorInfo);
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(errorInfo);
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
index ca99605f7,0917858f7..ed94e6771
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
@@@ -26,11 -31,11 +31,14 @@@ import org.apache.rocketmq.broker.clien
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.broker.filter.ConsumerFilterData;
import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
+ import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;
+ import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
+import org.apache.rocketmq.common.protocol.header.GetAllProducerInfoRequestHeader;
+import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;
import org.apache.rocketmq.common.AclConfig;
+ import org.apache.rocketmq.common.LockCallback;
import org.apache.rocketmq.common.MQVersion;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.PlainAccessConfig;
@@@ -104,9 -126,20 +129,19 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader;
import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+ import org.apache.rocketmq.common.rpc.RpcClientUtils;
+ import org.apache.rocketmq.common.rpc.RpcException;
+ import org.apache.rocketmq.common.rpc.RpcRequest;
+ import org.apache.rocketmq.common.rpc.RpcResponse;
+ import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
+ import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
import org.apache.rocketmq.common.stats.StatsItem;
import org.apache.rocketmq.common.stats.StatsSnapshot;
+ import org.apache.rocketmq.common.subscription.GroupForbidden;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
-import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.filter.util.BitsArray;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
@@@ -138,12 -172,14 +174,15 @@@ import java.util.List
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentMap;
+import org.apache.rocketmq.store.config.BrokerRole;
- public class AdminBrokerProcessor extends AsyncNettyRequestProcessor {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
- private final BrokerController brokerController;
+ import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
+
+ public class AdminBrokerProcessor implements NettyRequestProcessor {
+ private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ protected final BrokerController brokerController;
public AdminBrokerProcessor(final BrokerController brokerController) {
this.brokerController = brokerController;
@@@ -254,12 -372,11 +379,14 @@@
private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ if (validateSlave(response)) {
+ return response;
+ }
final CreateTopicRequestHeader requestHeader =
(CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class);
- log.info("updateAndCreateTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ LOGGER.info("Broker receive request to update or create topic={}, caller address={}",
+ requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
String topic = requestHeader.getTopic();
@@@ -691,11 -1164,9 +1178,12 @@@
private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request)
throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ if (validateSlave(response)) {
+ return response;
+ }
- log.info("updateAndCreateSubscriptionGroup called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+ LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}",
+ RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
SubscriptionGroupConfig config = RemotingSerializable.decode(request.getBody(), SubscriptionGroupConfig.class);
if (config != null) {
@@@ -1188,21 -1735,11 +1772,21 @@@
return response;
}
+ public RemotingCommand deleteExpiredCommitLog() {
- log.warn("invoke deleteExpiredCommitLog start.");
++ LOGGER.warn("invoke deleteExpiredCommitLog start.");
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ brokerController.getMessageStore().executeDeleteFilesManually();
- log.warn("invoke deleteExpiredCommitLog end.");
++ LOGGER.warn("invoke deleteExpiredCommitLog end.");
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
+
public RemotingCommand cleanUnusedTopic() {
- log.warn("invoke cleanUnusedTopic start.");
+ LOGGER.warn("invoke cleanUnusedTopic start.");
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
brokerController.getMessageStore().cleanUnusedTopic(brokerController.getTopicConfigManager().getTopicConfigTable().keySet());
- log.warn("invoke cleanUnusedTopic end.");
+ LOGGER.warn("invoke cleanUnusedTopic end.");
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
return response;
@@@ -1683,13 -2259,116 +2306,126 @@@
return inner;
}
+ private RemotingCommand getTopicConfig(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ GetTopicConfigRequestHeader requestHeader = (GetTopicConfigRequestHeader) request.decodeCommandCustomHeader(GetTopicConfigRequestHeader.class);
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(requestHeader.getTopic());
+ if (topicConfig == null) {
+ LOGGER.error("No topic in this broker, client: {} topic: {}", ctx.channel().remoteAddress(), requestHeader.getTopic());
+ //be care of the response code, should set "not-exist" explictly
+ response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+ response.setRemark("No topic in this broker. topic: " + requestHeader.getTopic());
+ return response;
+ }
+ TopicQueueMappingDetail topicQueueMappingDetail = null;
+ if (Boolean.TRUE.equals(requestHeader.getLo())) {
+ topicQueueMappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(requestHeader.getTopic());
+ }
+ String content = JSONObject.toJSONString(new TopicConfigAndQueueMapping(topicConfig, topicQueueMappingDetail));
+ try {
+ response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.error("UnsupportedEncodingException getTopicConfig: topic=" + topicConfig.getTopicName(), e);
+
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("UnsupportedEncodingException " + e.getMessage());
+ return response;
+ }
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+
+ return response;
+ }
+
+ private RemotingCommand notifyMinBrokerIdChange(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
- NotifyMinBrokerIdChangeRequestHeader requestHeader = request.decodeCommandCustomHeader(NotifyMinBrokerIdChangeRequestHeader.class);
++ NotifyMinBrokerIdChangeRequestHeader requestHeader = (NotifyMinBrokerIdChangeRequestHeader) request.decodeCommandCustomHeader(NotifyMinBrokerIdChangeRequestHeader.class);
+
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ LOGGER.warn("min broker id changed, prev {}, new {}", this.brokerController.getMinBrokerIdInGroup(), requestHeader.getMinBrokerId());
+
+ this.brokerController.updateMinBroker(requestHeader.getMinBrokerId(), requestHeader.getMinBrokerAddr(),
+ requestHeader.getOfflineBrokerAddr(),
+ requestHeader.getHaBrokerAddr());
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+
+ return response;
+ }
+
+ private RemotingCommand updateBrokerHaInfo(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ RemotingCommand response = RemotingCommand.createResponseCommand(ExchangeHAInfoResponseHeader.class);
+
- ExchangeHAInfoRequestHeader requestHeader = request.decodeCommandCustomHeader(ExchangeHAInfoRequestHeader.class);
++ ExchangeHAInfoRequestHeader requestHeader = (ExchangeHAInfoRequestHeader) request.decodeCommandCustomHeader(ExchangeHAInfoRequestHeader.class);
+ if (requestHeader.getMasterHaAddress() != null) {
+ this.brokerController.getMessageStore().updateHaMasterAddress(requestHeader.getMasterHaAddress());
+ this.brokerController.getMessageStore().updateMasterAddress(requestHeader.getMasterAddress());
+ if (this.brokerController.getMessageStore().getMasterFlushedOffset() == 0
+ && this.brokerController.getMessageStoreConfig().isSyncMasterFlushOffsetWhenStartup()) {
+ LOGGER.info("Set master flush offset in slave to {}", requestHeader.getMasterFlushOffset());
+ this.brokerController.getMessageStore().setMasterFlushedOffset(requestHeader.getMasterFlushOffset());
+ }
+ } else if (this.brokerController.getBrokerConfig().getBrokerId() == MixAll.MASTER_ID) {
+ final ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.readCustomHeader();
+ responseHeader.setMasterHaAddress(this.brokerController.getHAServerAddr());
+ responseHeader.setMasterFlushOffset(this.brokerController.getMessageStore().getBrokerInitMaxOffset());
+ responseHeader.setMasterAddress(this.brokerController.getBrokerAddr());
+ }
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+
+ return response;
+ }
+
+ private RemotingCommand getBrokerHaStatus(ChannelHandlerContext ctx, RemotingCommand request) {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ HARuntimeInfo runtimeInfo = this.brokerController.getMessageStore().getHARuntimeInfo();
+
+ if (runtimeInfo != null) {
+ byte[] body = runtimeInfo.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("Can not get HARuntimeInfo, may be duplicationEnable is true");
+ }
+
+ return response;
+ }
+
+ private RemotingCommand resetMasterFlushOffset(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ if (this.brokerController.getBrokerConfig().getBrokerId() != MixAll.MASTER_ID) {
+
- ResetMasterFlushOffsetHeader requestHeader = request.decodeCommandCustomHeader(ResetMasterFlushOffsetHeader.class);
++ ResetMasterFlushOffsetHeader requestHeader = (ResetMasterFlushOffsetHeader) request.decodeCommandCustomHeader(ResetMasterFlushOffsetHeader.class);
+
+ if (requestHeader.getMasterFlushOffset() != null) {
+ this.brokerController.getMessageStore().setMasterFlushedOffset(requestHeader.getMasterFlushOffset());
+ }
+ }
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
++
+ private boolean validateSlave(RemotingCommand response) {
+ if (this.brokerController.getMessageStoreConfig().getBrokerRole().equals(BrokerRole.SLAVE)) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("Can't modify topic or subscription group from slave broker, " +
+ "please execute it from master broker.");
+ return true;
+ }
+ return false;
+ }
}
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
index f0836f86e,7033417b5..0e8bac793
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
@@@ -75,6 -79,200 +79,200 @@@ public class PullMessageProcessor imple
public PullMessageProcessor(final BrokerController brokerController) {
this.brokerController = brokerController;
+ this.pullMessageResultHandler = new DefaultPullMessageResultHandler(brokerController);
+ }
+
+ private RemotingCommand rewriteRequestForStaticTopic(PullMessageRequestHeader requestHeader,
+ TopicQueueMappingContext mappingContext) {
+ try {
+ if (mappingContext.getMappingDetail() == null) {
+ return null;
+ }
+ TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+ String topic = mappingContext.getTopic();
+ Integer globalId = mappingContext.getGlobalId();
+ // if the leader? consider the order consumer, which will lock the mq
+ if (!mappingContext.isLeader()) {
+ return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d cannot find mapping item in request process of current broker %s", topic, globalId, mappingDetail.getBname()));
+ }
- Long globalOffset = requestHeader.getQueueOffset();
+
++ Long globalOffset = requestHeader.getQueueOffset();
+ LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), globalOffset, true);
+ mappingContext.setCurrentItem(mappingItem);
+
+ if (globalOffset < mappingItem.getLogicOffset()) {
+ //handleOffsetMoved
+ //If the physical queue is reused, we should handle the PULL_OFFSET_MOVED independently
+ //Otherwise, we could just transfer it to the physical process
+ }
+ //below are physical info
+ String bname = mappingItem.getBname();
+ Integer phyQueueId = mappingItem.getQueueId();
+ Long phyQueueOffset = mappingItem.computePhysicalQueueOffset(globalOffset);
+ requestHeader.setQueueId(phyQueueId);
+ requestHeader.setQueueOffset(phyQueueOffset);
+ if (mappingItem.checkIfEndOffsetDecided()
+ && requestHeader.getMaxMsgNums() != null) {
+ requestHeader.setMaxMsgNums((int) Math.min(mappingItem.getEndOffset() - mappingItem.getStartOffset(), requestHeader.getMaxMsgNums()));
+ }
+
+ if (mappingDetail.getBname().equals(bname)) {
+ //just let it go, do the local pull process
+ return null;
+ }
+
+ int sysFlag = requestHeader.getSysFlag();
+ requestHeader.setLo(false);
+ requestHeader.setBname(bname);
+ sysFlag = PullSysFlag.clearSuspendFlag(sysFlag);
+ sysFlag = PullSysFlag.clearCommitOffsetFlag(sysFlag);
+ requestHeader.setSysFlag(sysFlag);
+ RpcRequest rpcRequest = new RpcRequest(RequestCode.PULL_MESSAGE, requestHeader, null);
+ RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();
+ if (rpcResponse.getException() != null) {
+ throw rpcResponse.getException();
+ }
+
+ PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) rpcResponse.getHeader();
+ {
+ RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, rpcResponse.getCode());
+ if (rewriteResult != null) {
+ return rewriteResult;
+ }
+ }
+ return RpcClientUtils.createCommandForRpcResponse(rpcResponse);
+ } catch (Throwable t) {
+ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+ }
+ }
+
+ private RemotingCommand rewriteResponseForStaticTopic(PullMessageRequestHeader requestHeader,
+ PullMessageResponseHeader responseHeader,
+ TopicQueueMappingContext mappingContext, final int code) {
+ try {
+ if (mappingContext.getMappingDetail() == null) {
+ return null;
+ }
+ TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+ LogicQueueMappingItem leaderItem = mappingContext.getLeaderItem();
+
+ LogicQueueMappingItem currentItem = mappingContext.getCurrentItem();
+
+ LogicQueueMappingItem earlistItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true);
+
+ assert currentItem.getLogicOffset() >= 0;
+
+ long requestOffset = requestHeader.getQueueOffset();
+ long nextBeginOffset = responseHeader.getNextBeginOffset();
+ long minOffset = responseHeader.getMinOffset();
+ long maxOffset = responseHeader.getMaxOffset();
+ int responseCode = code;
+
+ //consider the following situations
+ // 1. read from slave, currently not supported
+ // 2. the middle queue is truncated because of deleting commitlog
+ if (code != ResponseCode.SUCCESS) {
+ //note the currentItem maybe both the leader and the earliest
+ boolean isRevised = false;
+ if (leaderItem.getGen() == currentItem.getGen()) {
+ //read the leader
+ if (requestOffset > maxOffset) {
+ //actually, we need do nothing, but keep the code structure here
+ if (code == ResponseCode.PULL_OFFSET_MOVED) {
+ responseCode = ResponseCode.PULL_OFFSET_MOVED;
+ nextBeginOffset = maxOffset;
+ } else {
+ //maybe current broker is the slave
+ responseCode = code;
+ }
+ } else if (requestOffset < minOffset) {
+ nextBeginOffset = minOffset;
+ responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+ } else {
+ responseCode = code;
+ }
+ }
+ //note the currentItem maybe both the leader and the earliest
+ if (earlistItem.getGen() == currentItem.getGen()) {
+ //read the earliest one
+ if (requestOffset < minOffset) {
+ if (code == ResponseCode.PULL_OFFSET_MOVED) {
+ responseCode = ResponseCode.PULL_OFFSET_MOVED;
+ nextBeginOffset = minOffset;
+ } else {
+ //maybe read from slave, but we still set it to moved
+ responseCode = ResponseCode.PULL_OFFSET_MOVED;
+ nextBeginOffset = minOffset;
+ }
+ } else if (requestOffset >= maxOffset) {
+ //just move to another item
+ LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);
+ if (nextItem != null) {
+ isRevised = true;
+ currentItem = nextItem;
+ nextBeginOffset = currentItem.getStartOffset();
+ minOffset = currentItem.getStartOffset();
+ maxOffset = minOffset;
+ responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+ } else {
+ //maybe the next one's logic offset is -1
+ responseCode = ResponseCode.PULL_NOT_FOUND;
+ }
+ } else {
+ //let it go
+ responseCode = code;
+ }
+ }
+
+ //read from the middle item, ignore the PULL_OFFSET_MOVED
+ if (!isRevised
+ && leaderItem.getGen() != currentItem.getGen()
+ && earlistItem.getGen() != currentItem.getGen()) {
+ if (requestOffset < minOffset) {
+ nextBeginOffset = minOffset;
+ responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+ } else if (requestOffset >= maxOffset) {
+ //just move to another item
+ LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);
+ if (nextItem != null) {
+ currentItem = nextItem;
+ nextBeginOffset = currentItem.getStartOffset();
+ minOffset = currentItem.getStartOffset();
+ maxOffset = minOffset;
+ responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+ } else {
+ //maybe the next one's logic offset is -1
+ responseCode = ResponseCode.PULL_NOT_FOUND;
+ }
+ } else {
+ responseCode = code;
+ }
+ }
+ }
+
+ //handle nextBeginOffset
+ //the next begin offset should no more than the end offset
+ if (currentItem.checkIfEndOffsetDecided()
+ && nextBeginOffset >= currentItem.getEndOffset()) {
+ nextBeginOffset = currentItem.getEndOffset();
+ }
+ responseHeader.setNextBeginOffset(currentItem.computeStaticQueueOffsetStrictly(nextBeginOffset));
+ //handle min offset
+ responseHeader.setMinOffset(currentItem.computeStaticQueueOffsetStrictly(Math.max(currentItem.getStartOffset(), minOffset)));
+ //handle max offset
+ responseHeader.setMaxOffset(Math.max(currentItem.computeStaticQueueOffsetStrictly(maxOffset),
+ TopicQueueMappingDetail.computeMaxOffsetFromMapping(mappingDetail, mappingContext.getGlobalId())));
+ //set the offsetDelta
+ responseHeader.setOffsetDelta(currentItem.computeOffsetDelta());
+
+ if (code != ResponseCode.SUCCESS) {
+ return RemotingCommand.createResponseCommandWithHeader(responseCode, responseHeader);
+ } else {
+ return null;
+ }
+ } catch (Throwable t) {
+ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+ }
}
@Override
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
index 133165b9f,a088973fb..183f64f38
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
@@@ -39,8 -39,9 +39,8 @@@ import org.apache.rocketmq.logging.Inte
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.exception.RemotingException;
-import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
- import org.apache.rocketmq.store.MessageExtBrokerInner;
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
index 2e009972d,437990c93..e94759500
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
@@@ -440,11 -252,13 +252,13 @@@ public class SendMessageProcessor exten
msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName);
+
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
- PutMessageResult putMessageResult = null;
- Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
+
+ // Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
+ boolean sendTransactionPrepareMessage = false;
- if (traFlag != null && Boolean.parseBoolean(traFlag)
+ if (Boolean.parseBoolean(traFlag)
&& !(msgInner.getReconsumeTimes() > 0 && msgInner.getDelayTimeLevel() > 0)) { //For client under version 4.6.1
if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {
response.setCode(ResponseCode.NO_PERMISSION);
diff --cc broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
index 35df22719,48709b3f4..e0436f1a7
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
@@@ -54,18 -60,11 +60,12 @@@ import org.apache.rocketmq.remoting.exc
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
- import org.apache.rocketmq.store.AppendMessageResult;
- import org.apache.rocketmq.store.AppendMessageStatus;
import org.apache.rocketmq.store.DefaultMessageStore;
- import org.apache.rocketmq.store.MappedFile;
- import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.MessageStore;
- import org.apache.rocketmq.store.PutMessageResult;
- import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.config.MessageStoreConfig;
- import org.apache.rocketmq.store.schedule.ScheduleMessageService;
+ import org.apache.rocketmq.store.logfile.DefaultMappedFile;
import org.apache.rocketmq.store.stats.BrokerStats;
import org.junit.Before;
import org.junit.Test;
diff --cc broker/src/test/java/org/apache/rocketmq/broker/schedule/ScheduleMessageServiceTest.java
index e0f4a492e,c743fca25..cff1374b7
--- a/broker/src/test/java/org/apache/rocketmq/broker/schedule/ScheduleMessageServiceTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/schedule/ScheduleMessageServiceTest.java
@@@ -37,14 -27,37 +27,37 @@@ import java.nio.ByteBuffer
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+ import java.util.Random;
import java.util.UUID;
+ import java.util.concurrent.ConcurrentHashMap;
+ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
+ import org.apache.rocketmq.broker.BrokerController;
+ import org.apache.rocketmq.broker.failover.EscapeBridge;
+ import org.apache.rocketmq.broker.util.HookUtils;
+ import org.apache.rocketmq.common.BrokerConfig;
+ import org.apache.rocketmq.common.UtilAll;
+ import org.apache.rocketmq.common.message.MessageDecoder;
+ import org.apache.rocketmq.common.message.MessageExt;
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+ import org.apache.rocketmq.store.ConsumeQueueExt;
+ import org.apache.rocketmq.store.DefaultMessageStore;
+ import org.apache.rocketmq.store.GetMessageResult;
+ import org.apache.rocketmq.store.GetMessageStatus;
+ import org.apache.rocketmq.store.MessageArrivingListener;
+ import org.apache.rocketmq.store.PutMessageResult;
+ import org.apache.rocketmq.store.config.MessageStoreConfig;
+ import org.apache.rocketmq.store.stats.BrokerStatsManager;
+ import org.junit.After;
+ import org.junit.Before;
+ import org.junit.Test;
+ import org.mockito.Mockito;
-import static org.apache.rocketmq.store.stats.BrokerStatsManager.BROKER_PUT_NUMS;
-import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_NUMS;
-import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_SIZE;
+import static org.apache.rocketmq.common.stats.Stats.BROKER_PUT_NUMS;
+import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_NUMS;
+import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_SIZE;
import static org.assertj.core.api.Assertions.assertThat;
-
+ import static org.junit.Assert.*;
public class ScheduleMessageServiceTest {
diff --cc client/pom.xml
index 2b3b2be93,1e3d23e98..d695183f5
--- a/client/pom.xml
+++ b/client/pom.xml
@@@ -54,6 -51,12 +49,10 @@@
<dependency>
<groupId>io.opentracing</groupId>
<artifactId>opentracing-mock</artifactId>
- <version>0.33.0</version>
- <scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
</project>
diff --cc client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AbstractAllocateMessageQueueStrategy.java
index 22ba5060c,bd03e1ff5..e0a050091
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AbstractAllocateMessageQueueStrategy.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AbstractAllocateMessageQueueStrategy.java
@@@ -25,19 -23,30 +25,19 @@@ import org.apache.rocketmq.client.log.C
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.logging.InternalLogger;
-/**
- * Cycle average Hashing queue algorithm
- */
-public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy {
- private InternalLogger log;
-
- public AllocateMessageQueueAveragelyByCircle() {
- log = ClientLogger.getLog();
- }
+public abstract class AbstractAllocateMessageQueueStrategy implements AllocateMessageQueueStrategy {
- private final InternalLogger log = ClientLogger.getLog();
- public AllocateMessageQueueAveragelyByCircle(InternalLogger log) {
- this.log = log;
- }
++ protected InternalLogger log = ClientLogger.getLog();
- @Override
- public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
+ public boolean check(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
- if (currentCID == null || currentCID.length() < 1) {
+ if (StringUtils.isEmpty(currentCID)) {
throw new IllegalArgumentException("currentCID is empty");
}
- if (mqAll == null || mqAll.isEmpty()) {
+ if (CollectionUtils.isEmpty(mqAll)) {
throw new IllegalArgumentException("mqAll is null or mqAll empty");
}
- if (cidAll == null || cidAll.isEmpty()) {
+ if (CollectionUtils.isEmpty(cidAll)) {
throw new IllegalArgumentException("cidAll is null or cidAll empty");
}
diff --cc client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
index 895f27757,4c31041e7..d8abd0a01
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
@@@ -18,13 -18,25 +18,23 @@@ package org.apache.rocketmq.client.cons
import java.util.ArrayList;
import java.util.List;
-import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+ import org.apache.rocketmq.client.log.ClientLogger;
-import org.apache.rocketmq.common.message.MessageQueue;
+ import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.common.message.MessageQueue;
/**
* Average Hashing queue algorithm
*/
-public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy {
- private InternalLogger log;
+public class AllocateMessageQueueAveragely extends AbstractAllocateMessageQueueStrategy {
+ public AllocateMessageQueueAveragely() {
+ log = ClientLogger.getLog();
+ }
+
+ public AllocateMessageQueueAveragely(InternalLogger log) {
+ this.log = log;
+ }
+
@Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
diff --cc client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
index d23350074,bd03e1ff5..c07ed6809
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
@@@ -18,13 -18,25 +18,23 @@@ package org.apache.rocketmq.client.cons
import java.util.ArrayList;
import java.util.List;
-import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+ import org.apache.rocketmq.client.log.ClientLogger;
-import org.apache.rocketmq.common.message.MessageQueue;
+ import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.common.message.MessageQueue;
/**
* Cycle average Hashing queue algorithm
*/
-public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy {
- private InternalLogger log;
+public class AllocateMessageQueueAveragelyByCircle extends AbstractAllocateMessageQueueStrategy {
+ public AllocateMessageQueueAveragelyByCircle() {
+ log = ClientLogger.getLog();
+ }
+
+ public AllocateMessageQueueAveragelyByCircle(InternalLogger log) {
+ this.log = log;
+ }
+
@Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
diff --cc client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
index 5d7e1685e,15b32d850..4739225cd
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
@@@ -79,7 -92,8 +92,9 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+ import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody;
+ import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueryConsumeTimeSpanBody;
import org.apache.rocketmq.common.protocol.body.QueryCorrectionOffsetBody;
@@@ -98,7 -118,7 +119,8 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader;
import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader;
import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.common.protocol.header.GetAllProducerInfoRequestHeader;
import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader;
import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseBody;
import org.apache.rocketmq.common.protocol.header.GetConsumeStatsInBrokerHeader;
@@@ -151,11 -180,16 +182,17 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader;
import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
+ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+ import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+ import org.apache.rocketmq.common.subscription.GroupForbidden;
+import org.apache.rocketmq.common.rpchook.StreamTypeRPCHook;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+ import org.apache.rocketmq.common.sysflag.PullSysFlag;
import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.InvokeCallback;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.RemotingClient;
@@@ -396,9 -469,9 +476,9 @@@ public class MQClientAPIImpl implement
clusterAclVersionInfo.setBrokerAddr(responseHeader.getBrokerAddr());
clusterAclVersionInfo.setAclConfigDataVersion(DataVersion.fromJson(responseHeader.getVersion(), DataVersion.class));
HashMap<String, Object> dataVersionMap = JSON.parseObject(responseHeader.getAllAclFileVersion(), HashMap.class);
- Map<String, DataVersion> allAclConfigDataVersion = new HashMap<String, DataVersion>();
+ Map<String, DataVersion> allAclConfigDataVersion = new HashMap<String, DataVersion>(dataVersionMap.size(), 1);
for (Map.Entry<String, Object> entry : dataVersionMap.entrySet()) {
- allAclConfigDataVersion.put(entry.getKey(),DataVersion.fromJson(JSON.toJSONString(entry.getValue()), DataVersion.class));
+ allAclConfigDataVersion.put(entry.getKey(), DataVersion.fromJson(JSON.toJSONString(entry.getValue()), DataVersion.class));
}
clusterAclVersionInfo.setAllAclConfigDataVersion(allAclConfigDataVersion);
return clusterAclVersionInfo;
@@@ -629,10 -710,22 +717,10 @@@
}
String addr = instance.findBrokerAddressInPublish(retryBrokerName);
log.warn("async send msg by retry {} times. topic={}, brokerAddr={}, brokerName={}", tmp, msg.getTopic(), addr,
- retryBrokerName, e);
+ retryBrokerName, e);
- try {
- request.setOpaque(RemotingCommand.createNewRequestId());
- sendMessageAsync(addr, retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance,
- timesTotal, curTimes, context, producer);
- } catch (InterruptedException e1) {
- onExceptionImpl(retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1,
- context, false, producer);
- } catch (RemotingTooMuchRequestException e1) {
- onExceptionImpl(retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1,
- context, false, producer);
- } catch (RemotingException e1) {
- producer.updateFaultItem(brokerName, 3000, true);
- onExceptionImpl(retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1,
- context, true, producer);
- }
+ request.setOpaque(RemotingCommand.createNewRequestId());
+ sendMessageAsync(addr, retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance,
- timesTotal, curTimes, context, producer);
++ timesTotal, curTimes, context, producer);
} else {
if (context != null) {
diff --cc client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
index c3626c38b,4aca8f337..18bd8b3e4
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
@@@ -311,10 -317,10 +317,10 @@@ public class ConsumeMessageConcurrently
// Wrap topic with namespace before sending back message.
msg.setTopic(this.defaultMQPushConsumer.withNamespace(msg.getTopic()));
try {
- this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, context.getMessageQueue().getBrokerName());
+ this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, this.defaultMQPushConsumer.queueWithNamespace(context.getMessageQueue()));
return true;
} catch (Exception e) {
- log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg.toString(), e);
+ log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg, e);
}
return false;
diff --cc client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
index 668f9b6b6,325ca4fa6..3e23485cc
--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
@@@ -879,14 -887,9 +883,14 @@@ public class DefaultMQProducerImpl impl
}
}
- throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
+ throw new MQClientException("The broker[" + brokerName + "] not exist", null);
}
+ public MQClientInstance getMqClientFactory() {
+ return mQClientFactory;
+ }
+
+ @Deprecated
public MQClientInstance getmQClientFactory() {
return mQClientFactory;
}
diff --cc client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
index c8446bdf1,ab312f7df..02445b31f
--- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
@@@ -25,11 -36,37 +36,34 @@@ import org.apache.rocketmq.client.produ
import org.apache.rocketmq.client.producer.SendCallback;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.client.producer.SendStatus;
+ import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.PlainAccessConfig;
+ import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.protocol.RequestCode;
+ import org.apache.rocketmq.common.message.MessageDecoder;
+ import org.apache.rocketmq.common.message.MessageExt;
+ import org.apache.rocketmq.common.message.MessageQueueAssignment;
+ import org.apache.rocketmq.common.message.MessageRequestMode;
import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.body.ClusterAclVersionInfo;
+ import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody;
+ import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
-import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseBody;
+ import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody;
+ import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetEarliestMsgStoretimeResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.PopMessageResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.SearchOffsetResponseHeader;
import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;
diff --cc client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java
index a60d88ea0,26e341475..67e6b7d53
--- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java
@@@ -43,9 -44,9 +41,8 @@@ import org.mockito.junit.MockitoJUnitRu
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
- import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
diff --cc common/pom.xml
index 3304640b2,cea58306a..1ff9111db
--- a/common/pom.xml
+++ b/common/pom.xml
@@@ -40,13 -40,9 +40,17 @@@
<groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
</project>
diff --cc common/src/main/java/org/apache/rocketmq/common/MQVersion.java
index 60ab6f8cb,847fd7436..544ce5f44
--- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
+++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
@@@ -18,7 -18,7 +18,7 @@@ package org.apache.rocketmq.common
public class MQVersion {
- public static final int CURRENT_VERSION = Version.V4_9_4.ordinal();
- public static final int CURRENT_VERSION = Version.V5_0_0_BETA_SNAPSHOT.ordinal();
++ public static final int CURRENT_VERSION = Version.V5_0_0_SNAPSHOT.ordinal();
public static String getVersionDesc(int value) {
int length = Version.values().length;
diff --cc common/src/main/java/org/apache/rocketmq/common/MixAll.java
index b13a09f92,638d03806..0a6bcf1ee
--- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java
+++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
@@@ -89,8 -89,11 +93,12 @@@ public class MixAll
public static final String REPLY_MESSAGE_FLAG = "reply";
public static final String LMQ_PREFIX = "%LMQ%";
public static final String MULTI_DISPATCH_QUEUE_SPLITTER = ",";
+ public static final String REQ_T = "ReqT";
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
+ public static final String LOGICAL_QUEUE_MOCK_BROKER_PREFIX = "__syslo__";
+ public static final String METADATA_SCOPE_GLOBAL = "__global__";
+ public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST = "__syslo__none__";
+ public static final String MULTI_PATH_SPLITTER = System.getProperty("rocketmq.broker.multiPathSplitter", ",");
public static String getWSAddr() {
String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP);
diff --cc common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
index b86780f7d,d6d4a8f4f..0ff544cbb
--- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
+++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
@@@ -348,10 -470,17 +471,18 @@@ public class MessageDecoder
byte[] body = new byte[bodyLen];
byteBuffer.get(body);
+ if (checkCRC) {
+ //crc body
+ int crc = UtilAll.crc32(body, 0, bodyLen);
+ if (crc != bodyCRC) {
+ throw new Exception("Msg crc is error!");
+ }
+ }
+
// uncompress body
if (deCompressBody && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) {
- body = UtilAll.uncompress(body);
+ Compressor compressor = CompressorFactory.getCompressor(MessageSysFlag.getCompressionType(sysFlag));
+ body = compressor.decompress(body);
}
msgExt.setBody(body);
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
index 9912382ff,1b2692da0..918b34077
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
@@@ -191,7 -204,37 +204,36 @@@ public class RequestCode
public static final int ADD_WRITE_PERM_OF_BROKER = 327;
-
+ public static final int GET_TOPIC_CONFIG = 351;
+
+ public static final int GET_SUBSCRIPTIONGROUP_CONFIG = 352;
+ public static final int UPDATE_AND_GET_GROUP_FORBIDDEN = 353;
+
+ public static final int LITE_PULL_MESSAGE = 361;
+
+ public static final int QUERY_ASSIGNMENT = 400;
+ public static final int SET_MESSAGE_REQUEST_MODE = 401;
+ public static final int GET_ALL_MESSAGE_REQUEST_MODE = 402;
+
+ public static final int UPDATE_AND_CREATE_STATIC_TOPIC = 513;
+
- /**
- * Below request codes are used by broker container,
- * these request codes are started with '9'.
- */
+ public static final int GET_BROKER_MEMBER_GROUP = 901;
+
+ public static final int ADD_BROKER = 902;
+
+ public static final int REMOVE_BROKER = 903;
+
+ public static final int BROKER_HEARTBEAT = 904;
+
+ public static final int NOTIFY_MIN_BROKER_ID_CHANGE = 905;
+
+ public static final int EXCHANGE_BROKER_HA_INFO = 906;
+
+ public static final int GET_BROKER_HA_STATUS = 907;
+
+ public static final int RESET_MASTER_FLUSH_OFFSET = 908;
++
+ public static final int GET_ALL_PRODUCER_INFO = 328;
+
+ public static final int DELETE_EXPIRED_COMMITLOG = 329;
}
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
index 02fac8e59,08790fc29..486efdfb5
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
@@@ -20,17 -20,12 +20,16 @@@
*/
package org.apache.rocketmq.common.protocol.header;
+import java.util.HashMap;
-
- import org.apache.rocketmq.remoting.CommandCustomHeader;
+ import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.FastCodesHeader;
-public class PullMessageRequestHeader extends TopicQueueRequestHeader {
+import io.netty.buffer.ByteBuf;
+
- public class PullMessageRequestHeader implements CommandCustomHeader, FastCodesHeader {
++public class PullMessageRequestHeader extends TopicQueueRequestHeader implements FastCodesHeader {
@CFNotNull
private String consumerGroup;
@CFNotNull
@@@ -57,79 -55,6 +59,115 @@@
public void checkFields() throws RemotingCommandException {
}
+ @Override
+ public void encode(ByteBuf out) {
+ writeIfNotNull(out, "consumerGroup", consumerGroup);
+ writeIfNotNull(out, "topic", topic);
+ writeIfNotNull(out, "queueId", queueId);
+ writeIfNotNull(out, "queueOffset", queueOffset);
+ writeIfNotNull(out, "maxMsgNums", maxMsgNums);
+ writeIfNotNull(out, "sysFlag", sysFlag);
+ writeIfNotNull(out, "commitOffset", commitOffset);
+ writeIfNotNull(out, "suspendTimeoutMillis", suspendTimeoutMillis);
+ writeIfNotNull(out, "subscription", subscription);
+ writeIfNotNull(out, "subVersion", subVersion);
+ writeIfNotNull(out, "expressionType", expressionType);
++ writeIfNotNull(out, "maxMsgBytes", maxMsgBytes);
++ writeIfNotNull(out, "lo", lo);
++ writeIfNotNull(out, "ns", ns);
++ writeIfNotNull(out, "nsd", nsd);
++ writeIfNotNull(out, "bname", bname);
++ writeIfNotNull(out, "oway", oway);
+ }
+
+ @Override
+ public void decode(HashMap<String, String> fields) throws RemotingCommandException {
+ String str = getAndCheckNotNull(fields, "consumerGroup");
+ if (str != null) {
+ this.consumerGroup = str;
+ }
+
+ str = getAndCheckNotNull(fields, "topic");
+ if (str != null) {
+ this.topic = str;
+ }
+
+ str = getAndCheckNotNull(fields, "queueId");
+ if (str != null) {
+ this.queueId = Integer.parseInt(str);
+ }
+
+ str = getAndCheckNotNull(fields, "queueOffset");
+ if (str != null) {
+ this.queueOffset = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "maxMsgNums");
+ if (str != null) {
+ this.maxMsgNums = Integer.parseInt(str);
+ }
+
+ str = getAndCheckNotNull(fields, "sysFlag");
+ if (str != null) {
+ this.sysFlag = Integer.parseInt(str);
+ }
+
+ str = getAndCheckNotNull(fields, "commitOffset");
+ if (str != null) {
+ this.commitOffset = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "suspendTimeoutMillis");
+ if (str != null) {
+ this.suspendTimeoutMillis = Long.parseLong(str);
+ }
+
+ str = fields.get("subscription");
+ if (str != null) {
+ this.subscription = str;
+ }
+
+ str = getAndCheckNotNull(fields, "subVersion");
+ if (str != null) {
+ this.subVersion = Long.parseLong(str);
+ }
+
+ str = fields.get("expressionType");
+ if (str != null) {
+ this.expressionType = str;
+ }
++
++ str = fields.get("maxMsgBytes");
++ if (str != null) {
++ this.maxMsgBytes = Integer.parseInt(str);
++ }
++
++ str = fields.get("lo");
++ if (str != null) {
++ this.lo = Boolean.parseBoolean(str);
++ }
++
++ str = fields.get("ns");
++ if (str != null) {
++ this.ns = str;
++ }
++
++ str = fields.get("nsd");
++ if (str != null) {
++ this.nsd = Boolean.parseBoolean(str);
++ }
++
++ str = fields.get("bname");
++ if (str != null) {
++ this.bname = str;
++ }
++
++ str = fields.get("oway");
++ if (str != null) {
++ this.oway = Boolean.parseBoolean(str);
++ }
+ }
+
public String getConsumerGroup() {
return consumerGroup;
}
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java
index 580d22277,84e6daf12..fb92a72d6
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java
@@@ -20,16 -20,12 +20,17 @@@
*/
package org.apache.rocketmq.common.protocol.header;
+import java.util.HashMap;
+
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
+ import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.FastCodesHeader;
+
+import io.netty.buffer.ByteBuf;
-public class PullMessageResponseHeader implements CommandCustomHeader {
+public class PullMessageResponseHeader implements CommandCustomHeader, FastCodesHeader {
@CFNotNull
private Long suggestWhichBrokerId;
@CFNotNull
@@@ -43,37 -47,6 +52,62 @@@
public void checkFields() throws RemotingCommandException {
}
+ @Override
+ public void encode(ByteBuf out) {
+ writeIfNotNull(out, "suggestWhichBrokerId", suggestWhichBrokerId);
+ writeIfNotNull(out, "nextBeginOffset", nextBeginOffset);
+ writeIfNotNull(out, "minOffset", minOffset);
+ writeIfNotNull(out, "maxOffset", maxOffset);
++ writeIfNotNull(out, "offsetDelta", offsetDelta);
++ writeIfNotNull(out, "topicSysFlag", topicSysFlag);
++ writeIfNotNull(out, "groupSysFlag", groupSysFlag);
++ writeIfNotNull(out, "forbiddenType", forbiddenType);
+ }
+
+ @Override
+ public void decode(HashMap<String, String> fields) throws RemotingCommandException {
+ String str = getAndCheckNotNull(fields, "suggestWhichBrokerId");
+ if (str != null) {
+ this.suggestWhichBrokerId = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "nextBeginOffset");
+ if (str != null) {
+ this.nextBeginOffset = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "minOffset");
+ if (str != null) {
+ this.minOffset = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "maxOffset");
+ if (str != null) {
+ this.maxOffset = Long.parseLong(str);
+ }
++
++ str = getAndCheckNotNull(fields, "offsetDelta");
++ if (str != null) {
++ this.offsetDelta = Long.parseLong(str);
++ }
++
++ str = getAndCheckNotNull(fields, "topicSysFlag");
++ if (str != null) {
++ this.topicSysFlag = Integer.parseInt(str);
++ }
++
++ str = getAndCheckNotNull(fields, "groupSysFlag");
++ if (str != null) {
++ this.groupSysFlag = Integer.parseInt(str);
++ }
++
++ str = getAndCheckNotNull(fields, "forbiddenType");
++ if (str != null) {
++ this.forbiddenType = Integer.parseInt(str);
++ }
++
+ }
+
public Long getNextBeginOffset() {
return nextBeginOffset;
}
@@@ -105,4 -78,39 +139,36 @@@
public void setSuggestWhichBrokerId(Long suggestWhichBrokerId) {
this.suggestWhichBrokerId = suggestWhichBrokerId;
}
+
+ public Integer getTopicSysFlag() {
+ return topicSysFlag;
+ }
+
+ public void setTopicSysFlag(Integer topicSysFlag) {
+ this.topicSysFlag = topicSysFlag;
+ }
+
+ public Integer getGroupSysFlag() {
+ return groupSysFlag;
+ }
+
+ public void setGroupSysFlag(Integer groupSysFlag) {
+ this.groupSysFlag = groupSysFlag;
+ }
- /**
- * @return the forbiddenType
- */
++
+ public Integer getForbiddenType() {
+ return forbiddenType;
+ }
- /**
- */
++
+ public void setForbiddenType(Integer forbiddenType) {
+ this.forbiddenType = forbiddenType;
+ }
+
+ public Long getOffsetDelta() {
+ return offsetDelta;
+ }
+
+ public void setOffsetDelta(Long offsetDelta) {
+ this.offsetDelta = offsetDelta;
+ }
}
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageResponseHeader.java
index 224a5264a,601f720e4..c6eaaee6b
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageResponseHeader.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageResponseHeader.java
@@@ -42,37 -38,6 +43,43 @@@ public class SendMessageResponseHeader
public void checkFields() throws RemotingCommandException {
}
+ @Override
+ public void encode(ByteBuf out) {
+ writeIfNotNull(out, "msgId", msgId);
+ writeIfNotNull(out, "queueId", queueId);
+ writeIfNotNull(out, "queueOffset", queueOffset);
+ writeIfNotNull(out, "transactionId", transactionId);
++ writeIfNotNull(out, "batchUniqId", batchUniqId);
+ }
+
+ @Override
+ public void decode(HashMap<String, String> fields) throws RemotingCommandException {
+ String str = getAndCheckNotNull(fields, "msgId");
+ if (str != null) {
+ this.msgId = str;
+ }
+
+ str = getAndCheckNotNull(fields, "queueId");
+ if (str != null) {
+ this.queueId = Integer.parseInt(str);
+ }
+
+ str = getAndCheckNotNull(fields, "queueOffset");
+ if (str != null) {
+ this.queueOffset = Long.parseLong(str);
+ }
+
+ str = fields.get("transactionId");
+ if (str != null) {
+ this.transactionId = str;
+ }
++
++ str = fields.get("batchUniqId");
++ if (str != null) {
++ this.batchUniqId = str;
++ }
+ }
+
public String getMsgId() {
return msgId;
}
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
index 50c789363,9a67ca5dc..6bca94d8d
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
@@@ -20,9 -20,8 +20,10 @@@ package org.apache.rocketmq.common.prot
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+ import java.util.Map;
import java.util.Random;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.MixAll;
public class BrokerData implements Comparable<BrokerData> {
@@@ -42,9 -58,17 +60,17 @@@
this.brokerAddrs = brokerAddrs;
}
+ public BrokerData(String cluster, String brokerName, HashMap<Long, String> brokerAddrs,
+ boolean enableActingMaster) {
+ this.cluster = cluster;
+ this.brokerName = brokerName;
+ this.brokerAddrs = brokerAddrs;
+ this.enableActingMaster = enableActingMaster;
+ }
+
/**
-- * Selects a (preferably master) broker address from the registered list.
-- * If the master's address cannot be found, a slave broker address is selected in a random manner.
++ * Selects a (preferably master) broker address from the registered list. If the master's address cannot be found, a
++ * slave broker address is selected in a random manner.
*
* @return Broker address.
*/
@@@ -86,19 -118,28 +120,24 @@@
@Override
public boolean equals(Object obj) {
- if (this == obj)
+ if (this == obj) {
return true;
- if (obj == null)
+ }
+ if (obj == null) {
return false;
- if (getClass() != obj.getClass())
+ }
+ if (getClass() != obj.getClass()) {
return false;
+ }
BrokerData other = (BrokerData) obj;
if (brokerAddrs == null) {
- if (other.brokerAddrs != null)
+ if (other.brokerAddrs != null) {
return false;
- } else if (!brokerAddrs.equals(other.brokerAddrs))
+ }
+ } else if (!brokerAddrs.equals(other.brokerAddrs)) {
return false;
+ }
- if (brokerName == null) {
- return other.brokerName == null;
- } else {
- return brokerName.equals(other.brokerName);
- }
+ return StringUtils.equals(brokerName, other.brokerName);
}
@Override
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
index 2d900398e,0c02078eb..a53b8eebd
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
@@@ -33,21 -69,47 +69,49 @@@ public class TopicRouteData extends Rem
public TopicRouteData cloneTopicRouteData() {
TopicRouteData topicRouteData = new TopicRouteData();
-
+ topicRouteData.setQueueDatas(new ArrayList<>());
+ topicRouteData.setBrokerDatas(new ArrayList<>());
+ topicRouteData.setFilterServerTable(new HashMap<>());
topicRouteData.setOrderTopicConf(this.orderTopicConf);
- if (this.queueDatas != null) {
- topicRouteData.getQueueDatas().addAll(this.queueDatas);
+ topicRouteData.getQueueDatas().addAll(this.queueDatas);
+ topicRouteData.getBrokerDatas().addAll(this.brokerDatas);
+ topicRouteData.getFilterServerTable().putAll(this.filterServerTable);
+ if (this.topicQueueMappingByBroker != null) {
+ Map<String, TopicQueueMappingInfo> cloneMap = new HashMap<>(this.topicQueueMappingByBroker);
+ topicRouteData.setTopicQueueMappingByBroker(cloneMap);
+ }
+ return topicRouteData;
+ }
+
+ public TopicRouteData deepCloneTopicRouteData() {
+ TopicRouteData topicRouteData = new TopicRouteData();
+
+ topicRouteData.setOrderTopicConf(this.orderTopicConf);
+
+ for (final QueueData queueData : this.queueDatas) {
+ topicRouteData.getQueueDatas().add(new QueueData(queueData));
}
- if (this.brokerDatas != null) {
- topicRouteData.getBrokerDatas().addAll(this.brokerDatas);
+ for (final BrokerData brokerData : this.brokerDatas) {
+ topicRouteData.getBrokerDatas().add(new BrokerData(brokerData));
}
- if (this.filterServerTable != null) {
- topicRouteData.getFilterServerTable().putAll(this.filterServerTable);
+ for (final Map.Entry<String, List<String>> listEntry : this.filterServerTable.entrySet()) {
+ topicRouteData.getFilterServerTable().put(listEntry.getKey(),
+ new ArrayList<String>(listEntry.getValue()));
+ }
+ if (this.topicQueueMappingByBroker != null) {
+ Map<String, TopicQueueMappingInfo> cloneMap = new HashMap<>(this.topicQueueMappingByBroker.size());
+ for (final Map.Entry<String, TopicQueueMappingInfo> entry : this.getTopicQueueMappingByBroker().entrySet()) {
+ TopicQueueMappingInfo topicQueueMappingInfo = new TopicQueueMappingInfo(entry.getValue().getTopic(), entry.getValue().getTotalQueues(), entry.getValue().getBname(), entry.getValue().getEpoch());
+ topicQueueMappingInfo.setDirty(entry.getValue().isDirty());
+ topicQueueMappingInfo.setScope(entry.getValue().getScope());
+ ConcurrentMap<Integer, Integer> concurrentMap = new ConcurrentHashMap<Integer, Integer>(entry.getValue().getCurrIdMap());
+ topicQueueMappingInfo.setCurrIdMap(concurrentMap);
+ cloneMap.put(entry.getKey(), topicQueueMappingInfo);
+ }
+ topicRouteData.setTopicQueueMappingByBroker(cloneMap);
}
return topicRouteData;
diff --cc common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
index 2373cde12,d28ac621c..bf6060214
--- a/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
+++ b/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
@@@ -16,21 -16,7 +16,20 @@@
*/
package org.apache.rocketmq.common.sysflag;
+import org.apache.rocketmq.common.compression.CompressionType;
+
public class MessageSysFlag {
+
+ /**
+ * Meaning of each bit in the system flag
+ *
+ * | bit | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+ * |--------|---|---|-----------|----------|-------------|------------------|------------------|------------------|
+ * | byte 1 | | | STOREHOST | BORNHOST | TRANSACTION | TRANSACTION | MULTI_TAGS | COMPRESSED |
+ * | byte 2 | | | | | | COMPRESSION_TYPE | COMPRESSION_TYPE | COMPRESSION_TYPE |
+ * | byte 3 | | | | | | | | |
+ * | byte 4 | | | | | | | | |
- *
+ */
public final static int COMPRESSED_FLAG = 0x1;
public final static int MULTI_TAGS_FLAG = 0x1 << 1;
public final static int TRANSACTION_NOT_TYPE = 0;
@@@ -61,9 -40,7 +60,13 @@@
return flag & (~COMPRESSED_FLAG);
}
+ // To match the compression type
+ public static CompressionType getCompressionType(final int flag) {
+ return CompressionType.findByValue((flag & COMPRESSION_TYPE_COMPARATOR) >> 8);
+ }
+
+ public static boolean check(int flag, int expectedFlag) {
+ return (flag & expectedFlag) != 0;
+ }
++
}
diff --cc common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
index 0448ceb11,72857bc4a..83703fc10
--- a/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
@@@ -14,11 -14,12 +14,13 @@@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
- package org.apache.rocketmq.broker.util;
+
+ package org.apache.rocketmq.common.utils;
+import java.nio.charset.StandardCharsets;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
+ import org.apache.rocketmq.common.constant.LoggerName;
+ import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.logging.InternalLoggerFactory;
import java.io.BufferedReader;
import java.io.InputStream;
@@@ -36,20 -36,22 +37,19 @@@ public class ServiceProvider
private static ClassLoader thisClassLoader;
/**
-- * JDK1.3+ <a href= "http://java.sun.com/j2se/1.3/docs/guide/jar/jar.html#Service%20Provider" > 'Service Provider' specification</a>.
++ * JDK1.3+ <a href= "http://java.sun.com/j2se/1.3/docs/guide/jar/jar.html#Service%20Provider" > 'Service Provider'
++ * specification</a>.
*/
public static final String TRANSACTION_SERVICE_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService";
public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener";
+ public static final String HA_SERVICE_ID = "META-INF/service/org.apache.rocketmq.store.ha.HAService";
-
public static final String RPC_HOOK_ID = "META-INF/service/org.apache.rocketmq.remoting.RPCHook";
--
public static final String ACL_VALIDATOR_ID = "META-INF/service/org.apache.rocketmq.acl.AccessValidator";
--
--
static {
thisClassLoader = getClassLoader(ServiceProvider.class);
}
@@@ -57,7 -59,7 +57,8 @@@
/**
* Returns a string that uniquely identifies the specified object, including its class.
* <p>
-- * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString() method, but works even when the specified object's class has overidden the toString method.
++ * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString()
++ * method, but works even when the specified object's class has overidden the toString method.
*
* @param o may be null.
* @return a string of form classname@hashcode, or "null" if param o is null.
@@@ -120,7 -126,7 +121,7 @@@
names.add(serviceName);
}
-- services.add((T)initService(getContextClassLoader(), serviceName, clazz));
++ services.add((T) initService(getContextClassLoader(), serviceName, clazz));
serviceName = reader.readLine();
}
@@@ -170,10 -180,10 +171,10 @@@
// This indicates a problem with the ClassLoader tree. An incompatible ClassLoader was used to load the implementation.
LOG.error(
"Class {} loaded from classloader {} does not extend {} as loaded by this classloader.",
- new Object[] {serviceClazz.getName(),
- objectId(serviceClazz.getClassLoader()), clazz.getName()});
+ serviceClazz.getName(),
+ objectId(serviceClazz.getClassLoader()), clazz.getName());
}
- return (T)serviceClazz.getDeclaredConstructor().newInstance();
- return (T)serviceClazz.newInstance();
++ return (T) serviceClazz.getDeclaredConstructor().newInstance();
} catch (ClassNotFoundException ex) {
if (classLoader == thisClassLoader) {
// Nothing more to try, onwards.
@@@ -196,6 -206,6 +197,6 @@@
} catch (Exception e) {
LOG.error("Unable to init service.", e);
}
-- return (T)serviceClazz;
++ return (T) serviceClazz;
}
}
diff --cc common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java
index 1734cbdf7,1734cbdf7..6b0f75acb
--- a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java
+++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java
@@@ -22,8 -22,8 +22,6 @@@ import org.junit.Test
import static org.assertj.core.api.Assertions.assertThat;
--import java.nio.charset.StandardCharsets;
--
public class MessageClientIDSetterTest {
@Test
diff --cc container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
index 000000000,2512a3307..c684f0eb8
mode 000000,100644..100644
--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
+++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
@@@ -1,0 -1,280 +1,280 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.rocketmq.container;
+
+ import io.netty.channel.ChannelHandlerContext;
+
+ import java.io.UnsupportedEncodingException;
+ import java.util.List;
+ import java.util.Properties;
+ import org.apache.rocketmq.broker.BrokerController;
+ import org.apache.rocketmq.broker.BrokerStartup;
+ import org.apache.rocketmq.common.BrokerConfig;
+ import org.apache.rocketmq.common.BrokerIdentity;
+ import org.apache.rocketmq.common.MixAll;
+ import org.apache.rocketmq.common.constant.LoggerName;
+ import org.apache.rocketmq.common.protocol.RequestCode;
+ import org.apache.rocketmq.common.protocol.ResponseCode;
+ import org.apache.rocketmq.common.protocol.header.AddBrokerRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.GetBrokerConfigResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.RemoveBrokerRequestHeader;
+ import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.logging.InternalLoggerFactory;
+ import org.apache.rocketmq.remoting.common.RemotingHelper;
+ import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+ import org.apache.rocketmq.store.config.MessageStoreConfig;
+
+ public class BrokerContainerProcessor implements NettyRequestProcessor {
+ private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ private final BrokerContainer brokerContainer;
+ private List<BrokerBootHook> brokerBootHookList;
+
+ public BrokerContainerProcessor(BrokerContainer brokerContainer) {
+ this.brokerContainer = brokerContainer;
+ }
+
+ @Override
+ public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {
+ switch (request.getCode()) {
+ case RequestCode.ADD_BROKER:
+ return this.addBroker(ctx, request);
+ case RequestCode.REMOVE_BROKER:
+ return this.removeBroker(ctx, request);
+ case RequestCode.GET_BROKER_CONFIG:
+ return this.getBrokerConfig(ctx, request);
+ case RequestCode.UPDATE_BROKER_CONFIG:
+ return this.updateBrokerConfig(ctx, request);
+ default:
+ break;
+ }
+ return null;
+ }
+
+ @Override public boolean rejectRequest() {
+ return false;
+ }
+
+ private synchronized RemotingCommand addBroker(ChannelHandlerContext ctx,
+ RemotingCommand request) throws Exception {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final AddBrokerRequestHeader requestHeader = request.decodeCommandCustomHeader(AddBrokerRequestHeader.class);
++ final AddBrokerRequestHeader requestHeader = (AddBrokerRequestHeader) request.decodeCommandCustomHeader(AddBrokerRequestHeader.class);
+
+ LOGGER.info("addBroker called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ Properties brokerProperties = null;
+ String configPath = requestHeader.getConfigPath();
+
+ if (configPath != null && !configPath.isEmpty()) {
+ BrokerStartup.SystemConfigFileHelper configFileHelper = new BrokerStartup.SystemConfigFileHelper();
+ configFileHelper.setFile(configPath);
+
+ try {
+ brokerProperties = configFileHelper.loadConfig();
+ } catch (Exception e) {
+ LOGGER.error("addBroker load config from {} failed, {}", configPath, e);
+ }
+ } else {
+ byte[] body = request.getBody();
+ if (body != null) {
+ String bodyStr = new String(body, MixAll.DEFAULT_CHARSET);
+ brokerProperties = MixAll.string2Properties(bodyStr);
+ }
+ }
+
+ if (brokerProperties == null) {
+ LOGGER.error("addBroker properties empty");
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("addBroker properties empty");
+ return response;
+ }
+
+ BrokerConfig brokerConfig = new BrokerConfig();
+ MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+ MixAll.properties2Object(brokerProperties, brokerConfig);
+ MixAll.properties2Object(brokerProperties, messageStoreConfig);
+
+ messageStoreConfig.setHaListenPort(brokerConfig.getListenPort() + 1);
+
+ if (configPath != null && !configPath.isEmpty()) {
+ brokerConfig.setBrokerConfigPath(configPath);
+ }
+
+ if (!messageStoreConfig.isEnableDLegerCommitLog()) {
+ switch (messageStoreConfig.getBrokerRole()) {
+ case ASYNC_MASTER:
+ case SYNC_MASTER:
+ brokerConfig.setBrokerId(MixAll.MASTER_ID);
+ break;
+ case SLAVE:
+ if (brokerConfig.getBrokerId() <= 0) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("slave broker id must be > 0");
+ return response;
+ }
+ break;
+ default:
+ break;
+
+ }
+
+ if (messageStoreConfig.getTotalReplicas() < messageStoreConfig.getInSyncReplicas()
+ || messageStoreConfig.getTotalReplicas() < messageStoreConfig.getMinInSyncReplicas()
+ || messageStoreConfig.getInSyncReplicas() < messageStoreConfig.getMinInSyncReplicas()) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("invalid replicas number");
+ return response;
+ }
+ }
+
+ BrokerController brokerController;
+ try {
+ brokerController = this.brokerContainer.addBroker(brokerConfig, messageStoreConfig);
+ } catch (Exception e) {
+ LOGGER.error("addBroker exception {}", e);
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(e.getMessage());
+ return response;
+ }
+ if (brokerController != null) {
+ brokerController.getConfiguration().registerConfig(brokerProperties);
+ try {
+ for (BrokerBootHook brokerBootHook : brokerBootHookList) {
+ brokerBootHook.executeBeforeStart(brokerController, brokerProperties);
+ }
+ brokerController.start();
+
+ for (BrokerBootHook brokerBootHook : brokerBootHookList) {
+ brokerBootHook.executeAfterStart(brokerController, brokerProperties);
+ }
+ } catch (Exception e) {
+ LOGGER.error("start broker exception {}", e);
+ BrokerIdentity brokerIdentity;
+ if (messageStoreConfig.isEnableDLegerCommitLog()) {
+ brokerIdentity = new BrokerIdentity(brokerConfig.getBrokerClusterName(),
+ brokerConfig.getBrokerName(), Integer.parseInt(messageStoreConfig.getdLegerSelfId().substring(1)));
+ } else {
+ brokerIdentity = new BrokerIdentity(brokerConfig.getBrokerClusterName(),
+ brokerConfig.getBrokerName(), brokerConfig.getBrokerId());
+ }
+ this.brokerContainer.removeBroker(brokerIdentity);
+ brokerController.shutdown();
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("start broker failed, " + e);
+ return response;
+ }
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("add broker return null");
+ }
+
+ return response;
+ }
+
+ private synchronized RemotingCommand removeBroker(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final RemoveBrokerRequestHeader requestHeader = request.decodeCommandCustomHeader(RemoveBrokerRequestHeader.class);
++ final RemoveBrokerRequestHeader requestHeader = (RemoveBrokerRequestHeader) request.decodeCommandCustomHeader(RemoveBrokerRequestHeader.class);
+
+ LOGGER.info("removeBroker called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ BrokerIdentity brokerIdentity = new BrokerIdentity(requestHeader.getBrokerClusterName(), requestHeader.getBrokerName(), requestHeader.getBrokerId());
+
+ BrokerController brokerController;
+ try {
+ brokerController = this.brokerContainer.removeBroker(brokerIdentity);
+ } catch (Exception e) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(e.getMessage());
+ return response;
+ }
+
+ if (brokerController != null) {
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.BROKER_NOT_EXIST);
+ response.setRemark("Broker not exist");
+ }
+ return response;
+ }
+
+ public void registerBrokerBootHook(List<BrokerBootHook> brokerBootHookList) {
+ this.brokerBootHookList = brokerBootHookList;
+ }
+
+ private RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ LOGGER.info("updateSharedBrokerConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ byte[] body = request.getBody();
+ if (body != null) {
+ try {
+ String bodyStr = new String(body, MixAll.DEFAULT_CHARSET);
+ Properties properties = MixAll.string2Properties(bodyStr);
+ if (properties != null) {
+ LOGGER.info("updateSharedBrokerConfig, new config: [{}] client: {} ", properties, ctx.channel().remoteAddress());
+ this.brokerContainer.getConfiguration().update(properties);
+ } else {
+ LOGGER.error("string2Properties error");
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("string2Properties error");
+ return response;
+ }
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.error("", e);
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("UnsupportedEncodingException " + e);
+ return response;
+ }
+ }
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
+
+ private RemotingCommand getBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {
+
+ final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerConfigResponseHeader.class);
+ final GetBrokerConfigResponseHeader responseHeader = (GetBrokerConfigResponseHeader) response.readCustomHeader();
+
+ String content = this.brokerContainer.getConfiguration().getAllConfigsFormatString();
+ if (content != null && content.length() > 0) {
+ try {
+ response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.error("", e);
+
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("UnsupportedEncodingException " + e);
+ return response;
+ }
+ }
+
+ responseHeader.setVersion(this.brokerContainer.getConfiguration().getDataVersionJson());
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
+ }
diff --cc namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
index 94797667d,5c9a9340d..09069eaf8
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
@@@ -517,40 -529,62 +529,61 @@@ public class DefaultRequestProcessor im
}
private RemotingCommand getUnitTopicList(ChannelHandlerContext ctx,
- RemotingCommand request) throws RemotingCommandException {
+ RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- TopicList unitTopics = this.namesrvController.getRouteInfoManager().getUnitTopics();
- byte[] body = unitTopics.encode();
+ boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();
+
+ if (enableTopicList) {
+ TopicList unitTopicList = this.namesrvController.getRouteInfoManager().getUnitTopics();
+ byte[] body = unitTopicList.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("disable");
+ }
- response.setBody(body);
- response.setCode(ResponseCode.SUCCESS);
- response.setRemark(null);
return response;
}
private RemotingCommand getHasUnitSubTopicList(ChannelHandlerContext ctx,
- RemotingCommand request) throws RemotingCommandException {
+ RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- TopicList hasUnitSubTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubTopicList();
- byte[] body = hasUnitSubTopicList.encode();
+ boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();
+
+ if (enableTopicList) {
+ TopicList hasUnitSubTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubTopicList();
+ byte[] body = hasUnitSubTopicList.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("disable");
+ }
- response.setBody(body);
- response.setCode(ResponseCode.SUCCESS);
- response.setRemark(null);
return response;
}
- private RemotingCommand getHasUnitSubUnUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request)
- throws RemotingCommandException {
+ private RemotingCommand getHasUnitSubUnUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- TopicList hasUnitSubUnUnitTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubUnUnitTopicList();
- byte[] body = hasUnitSubUnUnitTopicList.encode();
+ boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();
+
+ if (enableTopicList) {
+ TopicList hasUnitSubUnUnitTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubUnUnitTopicList();
+ byte[] body = hasUnitSubUnUnitTopicList.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("disable");
+ }
- response.setBody(body);
- response.setCode(ResponseCode.SUCCESS);
- response.setRemark(null);
return response;
}
diff --cc namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
index 400b7e743,aac1259ea..4b657627b
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
@@@ -52,20 -65,47 +65,47 @@@ import org.apache.rocketmq.remoting.pro
public class RouteInfoManager {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);
- private final static long BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2;
+ private final static long DEFAULT_BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2;
private final ReadWriteLock lock = new ReentrantReadWriteLock();
- private final HashMap<String/* topic */, Map<String /* brokerName */ , QueueData>> topicQueueTable;
- private final HashMap<String/* brokerName */, BrokerData> brokerAddrTable;
- private final HashMap<String/* clusterName */, Set<String/* brokerName */>> clusterAddrTable;
- private final HashMap<String/* brokerAddr */, BrokerLiveInfo> brokerLiveTable;
- private final HashMap<String/* brokerAddr */, List<String>/* Filter Server */> filterServerTable;
+ private final Map<String/* topic */, Map<String, QueueData>> topicQueueTable;
+ private final Map<String/* brokerName */, BrokerData> brokerAddrTable;
+ private final Map<String/* clusterName */, Set<String/* brokerName */>> clusterAddrTable;
+ private final Map<BrokerAddrInfo/* brokerAddr */, BrokerLiveInfo> brokerLiveTable;
+ private final Map<BrokerAddrInfo/* brokerAddr */, List<String>/* Filter Server */> filterServerTable;
+ private final Map<String/* topic */, Map<String/*brokerName*/, TopicQueueMappingInfo>> topicQueueMappingInfoTable;
+
+ private final BatchUnRegisterService unRegisterService;
+
+ private final NamesrvController namesrvController;
+ private final NamesrvConfig namesrvConfig;
+
+ public RouteInfoManager(final NamesrvConfig namesrvConfig, NamesrvController namesrvController) {
- this.topicQueueTable = new ConcurrentHashMap<String, Map<String, QueueData>>(1024);
- this.brokerAddrTable = new ConcurrentHashMap<String, BrokerData>(128);
- this.clusterAddrTable = new ConcurrentHashMap<String, Set<String>>(32);
- this.brokerLiveTable = new ConcurrentHashMap<BrokerAddrInfo, BrokerLiveInfo>(256);
- this.filterServerTable = new ConcurrentHashMap<BrokerAddrInfo, List<String>>(256);
- this.topicQueueMappingInfoTable = new ConcurrentHashMap<String, Map<String, TopicQueueMappingInfo>>(1024);
++ this.topicQueueTable = new ConcurrentHashMap<>(1024);
++ this.brokerAddrTable = new ConcurrentHashMap<>(128);
++ this.clusterAddrTable = new ConcurrentHashMap<>(32);
++ this.brokerLiveTable = new ConcurrentHashMap<>(256);
++ this.filterServerTable = new ConcurrentHashMap<>(256);
++ this.topicQueueMappingInfoTable = new ConcurrentHashMap<>(1024);
+ this.unRegisterService = new BatchUnRegisterService(this, namesrvConfig);
+ this.namesrvConfig = namesrvConfig;
+ this.namesrvController = namesrvController;
+ }
+
+ public void start() {
+ this.unRegisterService.start();
+ }
- public RouteInfoManager() {
- this.topicQueueTable = new HashMap<>(1024);
- this.brokerAddrTable = new HashMap<>(128);
- this.clusterAddrTable = new HashMap<>(32);
- this.brokerLiveTable = new HashMap<>(256);
- this.filterServerTable = new HashMap<>(256);
+ public void shutdown() {
+ this.unRegisterService.shutdown(true);
+ }
+
+ public boolean submitUnRegisterBrokerRequest(UnRegisterBrokerRequestHeader unRegisterRequest) {
+ return this.unRegisterService.submit(unRegisterRequest);
+ }
+
+ // For test only
+ int blockedUnRegisterRequests() {
+ return this.unRegisterService.queueLength();
}
public ClusterInfo getAllClusterInfo() {
@@@ -96,20 -167,19 +167,19 @@@
if (brokerNames != null
&& !brokerNames.isEmpty()) {
Map<String, QueueData> queueDataMap = this.topicQueueTable.get(topic);
- for (String brokerName : brokerNames) {
- final QueueData removedQD = queueDataMap.remove(brokerName);
- if (removedQD != null) {
- log.info("deleteTopic, remove one broker's topic {} {} {}", brokerName, topic,
- removedQD);
+ if (queueDataMap != null) {
+ for (String brokerName : brokerNames) {
+ final QueueData removedQD = queueDataMap.remove(brokerName);
+ if (removedQD != null) {
+ log.info("deleteTopic, remove one broker's topic {} {} {}", brokerName, topic,
+ removedQD);
+ }
}
-
if (queueDataMap.isEmpty()) {
- log.info("deleteTopic, remove the topic all queue {} {}", brokerName, topic);
+ log.info("deleteTopic, remove the topic all queue {} {}", clusterName, topic);
this.topicQueueTable.remove(topic);
- break;
}
}
-
}
} finally {
this.lock.writeLock().unlock();
diff --cc pom.xml
index b7b7a50e6,5f6c4ed70..acb9c7049
--- a/pom.xml
+++ b/pom.xml
@@@ -98,58 -98,6 +98,60 @@@
<!-- Compiler settings properties -->
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
+
+ <slf4j.version>1.7.7</slf4j.version>
+ <logback.version>1.2.10</logback.version>
- <commons-cli.version>1.2</commons-cli.version>
++ <commons-cli.version>1.4</commons-cli.version>
+ <netty.version>4.1.65.Final</netty.version>
+ <bcpkix-jdk15on.version>1.69</bcpkix-jdk15on.version>
+ <fastjson.version>1.2.69_noneautotype</fastjson.version>
+ <javassist.version>3.20.0-GA</javassist.version>
+ <jna.version>4.2.2</jna.version>
+ <commons-lang3.version>3.4</commons-lang3.version>
++ <commons-io.version>2.6</commons-io.version>
+ <guava.version>31.0.1-jre</guava.version>
+ <openmessaging.version>0.3.1-alpha</openmessaging.version>
+ <log4j.version>1.2.17</log4j.version>
+ <snakeyaml.version>1.30</snakeyaml.version>
+ <commons-codec.version>1.9</commons-codec.version>
+ <logging-log4j.version>2.17.1</logging-log4j.version>
+ <commons-validator.version>1.7</commons-validator.version>
+ <zstd-jni.version>1.5.2-2</zstd-jni.version>
+ <lz4-java.version>1.8.0</lz4-java.version>
+ <opentracing.version>0.33.0</opentracing.version>
+ <jaeger.version>1.6.0</jaeger.version>
+ <dleger.version>0.2.6</dleger.version>
+ <annotations-api.version>6.0.53</annotations-api.version>
+ <extra-enforcer-rules.version>1.0-beta-4</extra-enforcer-rules.version>
++ <concurrentlinkedhashmap-lru.version>1.4.2</concurrentlinkedhashmap-lru.version>
+
+ <!-- Test dependencies -->
+ <junit.version>4.13.2</junit.version>
+ <assertj-core.version>2.6.0</assertj-core.version>
+ <mockito-core.version>3.10.0</mockito-core.version>
+ <awaitility.version>4.1.0</awaitility.version>
+ <truth.version>0.30</truth.version>
+
+ <!-- Build plugin dependencies -->
+ <versions-maven-plugin.version>2.2</versions-maven-plugin.version>
+ <dependency-mediator-maven-plugin.version>1.0.2</dependency-mediator-maven-plugin.version>
+ <clirr-maven-plugin.version>2.7</clirr-maven-plugin.version>
+ <maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>
+ <maven-compiler-plugin.version>3.5.1</maven-compiler-plugin.version>
+ <maven-source-plugin.version>3.0.1</maven-source-plugin.version>
+ <maven-help-plugin.version>2.2</maven-help-plugin.version>
+ <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
+ <apache-rat-plugin.version>0.12</apache-rat-plugin.version>
+ <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>
+ <coveralls-maven-plugin.version>4.3.0</coveralls-maven-plugin.version>
+ <jacoco-maven-plugin.version>0.8.5</jacoco-maven-plugin.version>
+ <maven-surefire-plugin.version>2.19.1</maven-surefire-plugin.version>
+ <findbugs-maven-plugin.version>3.0.4</findbugs-maven-plugin.version>
+ <sonar-maven-plugin.version>3.0.2</sonar-maven-plugin.version>
+ <maven-assembly-plugin.version>3.0.0</maven-assembly-plugin.version>
+ <maven-javadoc-plugin.version>2.10.4</maven-javadoc-plugin.version>
+ <maven-failsafe-plugin.version>2.19.1</maven-failsafe-plugin.version>
+
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
<!-- Exclude all generated code -->
<sonar.jacoco.itReportPath>${project.basedir}/../test/target/jacoco-it.exec</sonar.jacoco.itReportPath>
@@@ -588,13 -569,23 +596,23 @@@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.4</version>
+ <version>${commons-lang3.version}</version>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
- <version>2.6</version>
++ <version>${commons-io.version}</version>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>19.0</version>
+ <version>${guava.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.googlecode.concurrentlinkedhashmap</groupId>
+ <artifactId>concurrentlinkedhashmap-lru</artifactId>
- <version>1.4.2</version>
++ <version>${concurrentlinkedhashmap-lru.version}</version>
+ </dependency>
<dependency>
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-api</artifactId>
diff --cc remoting/src/main/java/org/apache/rocketmq/remoting/protocol/FastCodesHeader.java
index 760040685,000000000..ebf993088
mode 100644,000000..100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/FastCodesHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/FastCodesHeader.java
@@@ -1,50 -1,0 +1,50 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.remoting.protocol;
+
+import java.util.HashMap;
+
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+
+import io.netty.buffer.ByteBuf;
+
+public interface FastCodesHeader {
+
+ default String getAndCheckNotNull(HashMap<String, String> fields, String field) {
+ String value = fields.get(field);
+ if (value == null) {
+ String headerClass = this.getClass().getSimpleName();
+ RemotingCommand.log.error("the custom field {}.{} is null", headerClass, field);
+ // no exception throws, keep compatible with RemotingCommand.decodeCommandCustomHeader
+ }
+ return value;
+ }
+
+ default void writeIfNotNull(ByteBuf out, String key, Object value) {
+ if (value != null) {
+ RocketMQSerializable.writeStr(out, true, key);
+ RocketMQSerializable.writeStr(out, false, value.toString());
+ }
+ }
+
- public void encode(ByteBuf out);
++ void encode(ByteBuf out);
+
+ void decode(HashMap<String, String> fields) throws RemotingCommandException;
+
+
+}
diff --cc remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
index b6523a3fb,7d614cffb..720beb030
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
@@@ -17,24 -17,24 +17,28 @@@
package org.apache.rocketmq.remoting.protocol;
import com.alibaba.fastjson.annotation.JSONField;
+ import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.logging.InternalLoggerFactory;
+ import org.apache.rocketmq.remoting.CommandCustomHeader;
+ import org.apache.rocketmq.remoting.annotation.CFNotNull;
+ import org.apache.rocketmq.remoting.common.RemotingHelper;
+ import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.nio.ByteBuffer;
+ import java.util.Arrays;
import java.util.HashMap;
+ import java.util.HashSet;
import java.util.Map;
+ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
- import org.apache.rocketmq.logging.InternalLogger;
- import org.apache.rocketmq.logging.InternalLoggerFactory;
- import org.apache.rocketmq.remoting.CommandCustomHeader;
- import org.apache.rocketmq.remoting.annotation.CFNotNull;
- import org.apache.rocketmq.remoting.common.RemotingHelper;
- import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
public class RemotingCommand {
public static final String SERIALIZE_TYPE_PROPERTY = "rocketmq.serialize.type";
public static final String SERIALIZE_TYPE_ENV = "ROCKETMQ_SERIALIZE_TYPE";
@@@ -114,6 -123,17 +127,18 @@@
return createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, "not set any response code", classHeader);
}
- public static RemotingCommand buildErrorResponse(int code, String remark, Class<? extends CommandCustomHeader> classHeader) {
++ public static RemotingCommand buildErrorResponse(int code, String remark,
++ Class<? extends CommandCustomHeader> classHeader) {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(classHeader);
+ response.setCode(code);
+ response.setRemark(remark);
+ return response;
+ }
+
+ public static RemotingCommand buildErrorResponse(int code, String remark) {
+ return buildErrorResponse(code, remark, null);
+ }
+
public static RemotingCommand createResponseCommand(int code, String remark,
Class<? extends CommandCustomHeader> classHeader) {
RemotingCommand cmd = new RemotingCommand();
@@@ -178,11 -190,9 +203,12 @@@
return length & 0xFFFFFF;
}
- private static RemotingCommand headerDecode(ByteBuf byteBuffer, int len, SerializeType type) throws RemotingCommandException {
- private static RemotingCommand headerDecode(byte[] headerData, SerializeType type) throws RemotingCommandException {
++ private static RemotingCommand headerDecode(ByteBuf byteBuffer, int len,
++ SerializeType type) throws RemotingCommandException {
switch (type) {
case JSON:
+ byte[] headerData = new byte[len];
+ byteBuffer.readBytes(headerData);
RemotingCommand resultJson = RemotingSerializable.decode(headerData, RemotingCommand.class);
resultJson.setSerializeTypeCurrentRPC(type);
return resultJson;
@@@ -239,16 -255,10 +265,16 @@@
this.customHeader = customHeader;
}
- public <T extends CommandCustomHeader> T decodeCommandCustomHeader(Class<T> classHeader) throws RemotingCommandException {
- T objectHeader;
+ public CommandCustomHeader decodeCommandCustomHeader(
- Class<? extends CommandCustomHeader> classHeader) throws RemotingCommandException {
++ Class<? extends CommandCustomHeader> classHeader) throws RemotingCommandException {
+ return decodeCommandCustomHeader(classHeader, true);
+ }
+
+ public CommandCustomHeader decodeCommandCustomHeader(Class<? extends CommandCustomHeader> classHeader,
- boolean useFastEncode) throws RemotingCommandException {
++ boolean useFastEncode) throws RemotingCommandException {
+ CommandCustomHeader objectHeader;
try {
- objectHeader = classHeader.newInstance();
+ objectHeader = classHeader.getDeclaredConstructor().newInstance();
} catch (InstantiationException e) {
return null;
} catch (IllegalAccessException e) {
diff --cc remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java
index 5ec7ac902,f7ef585d5..5617b2246
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java
@@@ -19,6 -19,11 +19,9 @@@ package org.apache.rocketmq.remoting.pr
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
+ import java.util.HashSet;
+ import java.util.Set;
+
-import com.alibaba.fastjson.JSON;
-import java.util.HashSet;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
diff --cc srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
index c2aaf9620,adf61adbd..d2c77c370
--- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
+++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
@@@ -23,9 -24,9 +24,8 @@@ import java.nio.file.Files
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
- import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
diff --cc store/src/main/java/org/apache/rocketmq/store/CommitLog.java
index efed87f67,62e9f59b7..5bb6dbaad
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
@@@ -16,13 -16,30 +16,30 @@@
*/
package org.apache.rocketmq.store;
-import java.net.Inet4Address;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.UnpooledByteBufAllocator;
+ import java.net.Inet6Address;
-import java.net.InetAddress;
+ import java.net.InetSocketAddress;
-import java.net.SocketAddress;
+ import java.nio.ByteBuffer;
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.LinkedList;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Optional;
+ import java.util.Set;
+ import java.util.concurrent.CompletableFuture;
+ import java.util.concurrent.ExecutionException;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.TimeoutException;
+ import java.util.function.Supplier;
+
+ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.ServiceThread;
+ import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
- import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
@@@ -76,34 -79,31 +79,31 @@@ public class CommitLog implements Swapp
protected final PutMessageLock putMessageLock;
+ protected final TopicQueueLock topicQueueLock;
+
private volatile Set<String> fullStorePaths = Collections.emptySet();
- protected final MultiDispatch multiDispatch;
private final FlushDiskWatcher flushDiskWatcher;
- public CommitLog(final DefaultMessageStore defaultMessageStore) {
- String storePath = defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog();
- if (storePath.contains(MessageStoreConfig.MULTI_PATH_SPLITTER)) {
- this.mappedFileQueue = new MultiPathMappedFileQueue(defaultMessageStore.getMessageStoreConfig(),
- defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
- defaultMessageStore.getAllocateMappedFileService(), this::getFullStorePaths);
+ protected int commitLogSize;
+
+ public CommitLog(final DefaultMessageStore messageStore) {
+ String storePath = messageStore.getMessageStoreConfig().getStorePathCommitLog();
+ if (storePath.contains(MixAll.MULTI_PATH_SPLITTER)) {
+ this.mappedFileQueue = new MultiPathMappedFileQueue(messageStore.getMessageStoreConfig(),
+ messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
+ messageStore.getAllocateMappedFileService(), this::getFullStorePaths);
} else {
this.mappedFileQueue = new MappedFileQueue(storePath,
- defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
- defaultMessageStore.getAllocateMappedFileService());
+ messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
+ messageStore.getAllocateMappedFileService());
}
- this.defaultMessageStore = defaultMessageStore;
-
- if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
- this.flushCommitLogService = new GroupCommitService();
- } else {
- this.flushCommitLogService = new FlushRealTimeService();
- }
+ this.defaultMessageStore = messageStore;
- this.commitLogService = new CommitRealTimeService();
+ this.flushManager = new DefaultFlushManager();
- this.appendMessageCallback = new DefaultAppendMessageCallback(messageStore.getMessageStoreConfig().getMaxMessageSize());
+ this.appendMessageCallback = new DefaultAppendMessageCallback();
putMessageThreadLocal = new ThreadLocal<PutMessageThreadLocal>() {
@Override
protected PutMessageThreadLocal initialValue() {
@@@ -125,12 -127,13 +127,17 @@@
return fullStorePaths;
}
+ public long getTotalSize() {
+ return this.mappedFileQueue.getTotalFileSize();
+ }
+
+ public ThreadLocal<PutMessageThreadLocal> getPutMessageThreadLocal() {
+ return putMessageThreadLocal;
+ }
+
public boolean load() {
boolean result = this.mappedFileQueue.load();
+ this.mappedFileQueue.checkSelf();
log.info("load commit log " + (result ? "OK" : "Failed"));
return result;
}
@@@ -604,18 -754,17 +758,26 @@@
return keyBuilder.toString();
}
+ public void setMappedFileQueueOffset(final long phyOffset) {
+ this.mappedFileQueue.setFlushedWhere(phyOffset);
+ this.mappedFileQueue.setCommittedWhere(phyOffset);
+ }
+
+ public void updateMaxMessageSize(PutMessageThreadLocal putMessageThreadLocal) {
+ // dynamically adjust maxMessageSize, but not support DLedger mode temporarily
+ int newMaxMessageSize = this.defaultMessageStore.getMessageStoreConfig().getMaxMessageSize();
+ if (newMaxMessageSize >= 10 &&
+ putMessageThreadLocal.getEncoder().getMaxMessageBodySize() != newMaxMessageSize) {
+ putMessageThreadLocal.getEncoder().updateEncoderBufferCapacity(newMaxMessageSize);
+ }
+ }
+
public CompletableFuture<PutMessageResult> asyncPutMessage(final MessageExtBrokerInner msg) {
// Set the storage time
- msg.setStoreTimestamp(System.currentTimeMillis());
+ if (!defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {
+ msg.setStoreTimestamp(System.currentTimeMillis());
+ }
+
// Set the message body BODY CRC (consider the most appropriate setting
// on the client)
msg.setBodyCRC(UtilAll.crc32(msg.getBody()));
@@@ -659,65 -786,110 +799,111 @@@
}
PutMessageThreadLocal putMessageThreadLocal = this.putMessageThreadLocal.get();
+ updateMaxMessageSize(putMessageThreadLocal);
- if (!multiDispatch.isMultiDispatchMsg(msg)) {
+ String topicQueueKey = generateKey(putMessageThreadLocal.getKeyBuilder(), msg);
+ long elapsedTimeInLock = 0;
+ MappedFile unlockMappedFile = null;
+ MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
+
+ long currOffset;
+ if (mappedFile == null) {
+ currOffset = 0;
+ } else {
+ currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();
+ }
+
+ boolean needHandleHA = needHandleHA(msg);
+ int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();
+
+ if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableSlaveActingMaster()) {
+ int inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),
+ this.defaultMessageStore.getHaService().inSyncSlaveNums(currOffset) + 1);
+ needAckNums = calcNeedAckNums(inSyncReplicas);
+ if (needAckNums > inSyncReplicas) {
+ // Tell the producer, don't have enough slaves to handle the send request
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));
+ }
+ }
+
+ topicQueueLock.lock(topicQueueKey);
+ try {
+
+ boolean needAssignOffset = true;
+ if (defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()
+ && defaultMessageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) {
+ needAssignOffset = false;
+ }
+ if (needAssignOffset) {
+ defaultMessageStore.assignOffset(msg, getMessageNum(msg));
+ }
+
PutMessageResult encodeResult = putMessageThreadLocal.getEncoder().encode(msg);
if (encodeResult != null) {
return CompletableFuture.completedFuture(encodeResult);
}
- msg.setEncodedBuff(putMessageThreadLocal.getEncoder().encoderBuffer);
+ msg.setEncodedBuff(putMessageThreadLocal.getEncoder().getEncoderBuffer());
- }
- PutMessageContext putMessageContext = new PutMessageContext(generateKey(putMessageThreadLocal.getKeyBuilder(), msg));
+ PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);
- long elapsedTimeInLock = 0;
- MappedFile unlockMappedFile = null;
+ putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
+ try {
+ long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
+ this.beginTimeInLock = beginLockTimestamp;
- putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
- try {
- MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
- long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
- this.beginTimeInLock = beginLockTimestamp;
+ // Here settings are stored timestamp, in order to ensure an orderly
+ // global
+ if (!defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {
+ msg.setStoreTimestamp(beginLockTimestamp);
+ }
- // Here settings are stored timestamp, in order to ensure an orderly
- // global
- msg.setStoreTimestamp(beginLockTimestamp);
+ if (null == mappedFile || mappedFile.isFull()) {
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
+ }
+ if (null == mappedFile) {
+ log.error("create mapped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null));
+ }
- if (null == mappedFile || mappedFile.isFull()) {
- mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
- }
- if (null == mappedFile) {
- log.error("create mapped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null));
- }
+ result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
+ switch (result.getStatus()) {
+ case PUT_OK:
+ onCommitLogAppend(msg, result, mappedFile);
+ break;
+ case END_OF_FILE:
+ onCommitLogAppend(msg, result, mappedFile);
+ unlockMappedFile = mappedFile;
+ // Create a new file, re-write the message
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0);
+ if (null == mappedFile) {
+ // XXX: warn and notify me
+ log.error("create mapped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, result));
+ }
+ result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
+ if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) {
+ onCommitLogAppend(msg, result, mappedFile);
+ }
+ break;
+ case MESSAGE_SIZE_EXCEEDED:
+ case PROPERTIES_SIZE_EXCEEDED:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
+ case UNKNOWN_ERROR:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
+ default:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
+ }
- result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
- switch (result.getStatus()) {
- case PUT_OK:
- break;
- case END_OF_FILE:
- unlockMappedFile = mappedFile;
- // Create a new file, re-write the message
- mappedFile = this.mappedFileQueue.getLastMappedFile(0);
- if (null == mappedFile) {
- // XXX: warn and notify me
- log.error("create mapped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result));
- }
- result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
- break;
- case MESSAGE_SIZE_EXCEEDED:
- case PROPERTIES_SIZE_EXCEEDED:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
- case UNKNOWN_ERROR:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
- default:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
+ elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
+ beginTimeInLock = 0;
+ } finally {
+ putMessageLock.unlock();
}
-
- elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
} finally {
- beginTimeInLock = 0;
- putMessageLock.unlock();
+ topicQueueLock.unlock(topicQueueKey);
}
if (elapsedTimeInLock > 500) {
@@@ -776,58 -938,90 +952,91 @@@
MappedFile unlockMappedFile = null;
MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
+ long currOffset;
+ if (mappedFile == null) {
+ currOffset = 0;
+ } else {
+ currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();
+ }
+
+ boolean needHandleHA = needHandleHA(messageExtBatch);
+ int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();
+
+ if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableSlaveActingMaster()) {
+ int inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),
+ this.defaultMessageStore.getHaService().inSyncSlaveNums(currOffset) + 1);
+ needAckNums = calcNeedAckNums(inSyncReplicas);
+ if (needAckNums > inSyncReplicas) {
+ // Tell the producer, don't have enough slaves to handle the send request
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));
+ }
+ }
+
//fine-grained lock instead of the coarse-grained
PutMessageThreadLocal pmThreadLocal = this.putMessageThreadLocal.get();
+ updateMaxMessageSize(pmThreadLocal);
MessageExtEncoder batchEncoder = pmThreadLocal.getEncoder();
- PutMessageContext putMessageContext = new PutMessageContext(generateKey(pmThreadLocal.getKeyBuilder(), messageExtBatch));
+ String topicQueueKey = generateKey(pmThreadLocal.getKeyBuilder(), messageExtBatch);
+
+ PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);
messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));
- putMessageLock.lock();
+ topicQueueLock.lock(topicQueueKey);
try {
- long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
- this.beginTimeInLock = beginLockTimestamp;
+ defaultMessageStore.assignOffset(messageExtBatch, (short) putMessageContext.getBatchSize());
- // Here settings are stored timestamp, in order to ensure an orderly
- // global
- messageExtBatch.setStoreTimestamp(beginLockTimestamp);
+ putMessageLock.lock();
+ try {
+ long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
+ this.beginTimeInLock = beginLockTimestamp;
- if (null == mappedFile || mappedFile.isFull()) {
- mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
- }
- if (null == mappedFile) {
- log.error("Create mapped file1 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null));
- }
+ // Here settings are stored timestamp, in order to ensure an orderly
+ // global
+ messageExtBatch.setStoreTimestamp(beginLockTimestamp);
- result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);
- switch (result.getStatus()) {
- case PUT_OK:
- break;
- case END_OF_FILE:
- unlockMappedFile = mappedFile;
- // Create a new file, re-write the message
- mappedFile = this.mappedFileQueue.getLastMappedFile(0);
- if (null == mappedFile) {
- // XXX: warn and notify me
- log.error("Create mapped file2 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result));
- }
- result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);
- break;
- case MESSAGE_SIZE_EXCEEDED:
- case PROPERTIES_SIZE_EXCEEDED:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
- case UNKNOWN_ERROR:
- default:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
- }
+ if (null == mappedFile || mappedFile.isFull()) {
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
+ }
+ if (null == mappedFile) {
+ log.error("Create mapped file1 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null));
+ }
- elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
+ result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);
+ switch (result.getStatus()) {
+ case PUT_OK:
+ break;
+ case END_OF_FILE:
+ unlockMappedFile = mappedFile;
+ // Create a new file, re-write the message
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0);
+ if (null == mappedFile) {
+ // XXX: warn and notify me
+ log.error("Create mapped file2 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, result));
+ }
+ result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);
+ break;
+ case MESSAGE_SIZE_EXCEEDED:
+ case PROPERTIES_SIZE_EXCEEDED:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
+ case UNKNOWN_ERROR:
+ default:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
+ }
+
+ elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
+ beginTimeInLock = 0;
+ } finally {
+ putMessageLock.unlock();
+ }
} finally {
- beginTimeInLock = 0;
- putMessageLock.unlock();
+ topicQueueLock.unlock(topicQueueKey);
}
if (elapsedTimeInLock > 500) {
@@@ -1527,13 -1869,8 +1865,15 @@@
final int msgLen = calMsgLength(msgInner.getSysFlag(), bodyLength, topicLength, propertiesLength);
+ // Exceeds the maximum message body
+ if (bodyLength > this.maxMessageBodySize) {
+ CommitLog.log.warn("message body size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength
+ + ", maxMessageSize: " + this.maxMessageBodySize);
+ return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+ }
+
+ final long queueOffset = msgInner.getQueueOffset();
+
// Exceeds the maximum message
if (msgLen > this.maxMessageSize) {
CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength
@@@ -1541,52 -1878,49 +1881,52 @@@
return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
}
- // Initialization of storage space
- this.resetByteBuffer(encoderBuffer, msgLen);
// 1 TOTALSIZE
- this.encoderBuffer.putInt(msgLen);
+ this.byteBuf.writeInt(msgLen);
// 2 MAGICCODE
- this.encoderBuffer.putInt(CommitLog.MESSAGE_MAGIC_CODE);
+ this.byteBuf.writeInt(CommitLog.MESSAGE_MAGIC_CODE);
// 3 BODYCRC
- this.encoderBuffer.putInt(msgInner.getBodyCRC());
+ this.byteBuf.writeInt(msgInner.getBodyCRC());
// 4 QUEUEID
- this.encoderBuffer.putInt(msgInner.getQueueId());
+ this.byteBuf.writeInt(msgInner.getQueueId());
// 5 FLAG
- this.encoderBuffer.putInt(msgInner.getFlag());
+ this.byteBuf.writeInt(msgInner.getFlag());
- // 6 QUEUEOFFSET, need update later
- this.byteBuf.writeLong(0);
+ // 6 QUEUEOFFSET
- this.encoderBuffer.putLong(queueOffset);
++ this.byteBuf.writeLong(queueOffset);
// 7 PHYSICALOFFSET, need update later
- this.encoderBuffer.putLong(0);
+ this.byteBuf.writeLong(0);
// 8 SYSFLAG
- this.encoderBuffer.putInt(msgInner.getSysFlag());
+ this.byteBuf.writeInt(msgInner.getSysFlag());
// 9 BORNTIMESTAMP
- this.encoderBuffer.putLong(msgInner.getBornTimestamp());
+ this.byteBuf.writeLong(msgInner.getBornTimestamp());
+
// 10 BORNHOST
- socketAddress2ByteBuffer(msgInner.getBornHost(), this.encoderBuffer);
+ ByteBuffer bornHostBytes = msgInner.getBornHostBytes();
+ this.byteBuf.writeBytes(bornHostBytes.array());
+
// 11 STORETIMESTAMP
- this.encoderBuffer.putLong(msgInner.getStoreTimestamp());
+ this.byteBuf.writeLong(msgInner.getStoreTimestamp());
+
// 12 STOREHOSTADDRESS
- socketAddress2ByteBuffer(msgInner.getStoreHost(), this.encoderBuffer);
+ ByteBuffer storeHostBytes = msgInner.getStoreHostBytes();
+ this.byteBuf.writeBytes(storeHostBytes.array());
+
// 13 RECONSUMETIMES
- this.encoderBuffer.putInt(msgInner.getReconsumeTimes());
+ this.byteBuf.writeInt(msgInner.getReconsumeTimes());
// 14 Prepared Transaction Offset
- this.encoderBuffer.putLong(msgInner.getPreparedTransactionOffset());
+ this.byteBuf.writeLong(msgInner.getPreparedTransactionOffset());
// 15 BODY
- this.encoderBuffer.putInt(bodyLength);
+ this.byteBuf.writeInt(bodyLength);
if (bodyLength > 0)
- this.encoderBuffer.put(msgInner.getBody());
+ this.byteBuf.writeBytes(msgInner.getBody());
// 16 TOPIC
- this.encoderBuffer.put((byte) topicLength);
- this.encoderBuffer.put(topicData);
+ this.byteBuf.writeByte((byte) topicLength);
+ this.byteBuf.writeBytes(topicData);
// 17 PROPERTIES
- this.encoderBuffer.putShort((short) propertiesLength);
+ this.byteBuf.writeShort((short) propertiesLength);
if (propertiesLength > 0)
- this.encoderBuffer.put(propertiesData);
+ this.byteBuf.writeBytes(propertiesData);
- encoderBuffer.flip();
return null;
}
@@@ -1639,54 -1973,63 +1979,54 @@@
final int topicLength = topicData.length;
int totalPropLen = needAppendLastPropertySeparator ? propertiesLen + batchPropLen + 1
- : propertiesLen + batchPropLen;
+ : propertiesLen + batchPropLen;
final int msgLen = calMsgLength(messageExtBatch.getSysFlag(), bodyLen, topicLength, totalPropLen);
- // Exceeds the maximum message
- if (msgLen > this.maxMessageSize) {
- CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLen
- + ", maxMessageSize: " + this.maxMessageSize);
- throw new RuntimeException("message size exceeded");
- }
-
- totalMsgLen += msgLen;
- // Determines whether there is sufficient free space
- if (totalMsgLen > maxMessageSize) {
- throw new RuntimeException("message size exceeded");
- }
-
// 1 TOTALSIZE
- this.encoderBuffer.putInt(msgLen);
+ this.byteBuf.writeInt(msgLen);
// 2 MAGICCODE
- this.encoderBuffer.putInt(CommitLog.MESSAGE_MAGIC_CODE);
+ this.byteBuf.writeInt(CommitLog.MESSAGE_MAGIC_CODE);
// 3 BODYCRC
- this.encoderBuffer.putInt(bodyCrc);
+ this.byteBuf.writeInt(bodyCrc);
// 4 QUEUEID
- this.encoderBuffer.putInt(messageExtBatch.getQueueId());
+ this.byteBuf.writeInt(messageExtBatch.getQueueId());
// 5 FLAG
- this.encoderBuffer.putInt(flag);
+ this.byteBuf.writeInt(flag);
// 6 QUEUEOFFSET
- this.encoderBuffer.putLong(0);
+ this.byteBuf.writeLong(0);
// 7 PHYSICALOFFSET
- this.encoderBuffer.putLong(0);
+ this.byteBuf.writeLong(0);
// 8 SYSFLAG
- this.encoderBuffer.putInt(messageExtBatch.getSysFlag());
+ this.byteBuf.writeInt(messageExtBatch.getSysFlag());
// 9 BORNTIMESTAMP
- this.encoderBuffer.putLong(messageExtBatch.getBornTimestamp());
+ this.byteBuf.writeLong(messageExtBatch.getBornTimestamp());
+
// 10 BORNHOST
- this.resetByteBuffer(bornHostHolder, bornHostLength);
- this.encoderBuffer.put(messageExtBatch.getBornHostBytes(bornHostHolder));
+ ByteBuffer bornHostBytes = messageExtBatch.getBornHostBytes();
+ this.byteBuf.writeBytes(bornHostBytes.array());
+
// 11 STORETIMESTAMP
- this.encoderBuffer.putLong(messageExtBatch.getStoreTimestamp());
+ this.byteBuf.writeLong(messageExtBatch.getStoreTimestamp());
+
// 12 STOREHOSTADDRESS
- this.resetByteBuffer(storeHostHolder, storeHostLength);
- this.encoderBuffer.put(messageExtBatch.getStoreHostBytes(storeHostHolder));
+ ByteBuffer storeHostBytes = messageExtBatch.getStoreHostBytes();
+ this.byteBuf.writeBytes(storeHostBytes.array());
+
// 13 RECONSUMETIMES
- this.encoderBuffer.putInt(messageExtBatch.getReconsumeTimes());
+ this.byteBuf.writeInt(messageExtBatch.getReconsumeTimes());
// 14 Prepared Transaction Offset, batch does not support transaction
- this.encoderBuffer.putLong(0);
+ this.byteBuf.writeLong(0);
// 15 BODY
- this.encoderBuffer.putInt(bodyLen);
+ this.byteBuf.writeInt(bodyLen);
if (bodyLen > 0)
- this.encoderBuffer.put(messagesByteBuff.array(), bodyPos, bodyLen);
+ this.byteBuf.writeBytes(messagesByteBuff.array(), bodyPos, bodyLen);
// 16 TOPIC
- this.encoderBuffer.put((byte) topicLength);
- this.encoderBuffer.put(topicData);
+ this.byteBuf.writeByte((byte) topicLength);
+ this.byteBuf.writeBytes(topicData);
// 17 PROPERTIES
- this.encoderBuffer.putShort((short) totalPropLen);
+ this.byteBuf.writeShort((short) totalPropLen);
if (propertiesLen > 0) {
- this.encoderBuffer.put(messagesByteBuff.array(), propertiesPos, propertiesLen);
+ this.byteBuf.writeBytes(messagesByteBuff.array(), propertiesPos, propertiesLen);
}
if (batchPropLen > 0) {
if (needAppendLastPropertySeparator) {
@@@ -1697,72 -2040,174 +2037,184 @@@
}
putMessageContext.setBatchSize(batchSize);
putMessageContext.setPhyPos(new long[batchSize]);
- encoderBuffer.flip();
- return encoderBuffer;
+
+ return this.byteBuf.nioBuffer();
}
- private void resetByteBuffer(final ByteBuffer byteBuffer, final int limit) {
- byteBuffer.flip();
- byteBuffer.limit(limit);
+ public ByteBuffer getEncoderBuffer() {
+ return this.byteBuf.nioBuffer();
}
+ public int getMaxMessageBodySize() {
+ return this.maxMessageBodySize;
+ }
+
+ public void updateEncoderBufferCapacity(int newMaxMessageBodySize) {
+ this.maxMessageBodySize = newMaxMessageBodySize;
+ //Reserve 64kb for encoding buffer outside body
+ this.maxMessageSize = Integer.MAX_VALUE - newMaxMessageBodySize >= 64 * 1024 ?
+ this.maxMessageBodySize + 64 * 1024 : Integer.MAX_VALUE;
+ this.byteBuf.capacity(this.maxMessageSize);
+ }
}
- static class PutMessageThreadLocal {
- private MessageExtEncoder encoder;
- private StringBuilder keyBuilder;
+ interface FlushManager {
+ void start();
- PutMessageThreadLocal(int maxMessageBodySize) {
- encoder = new MessageExtEncoder(maxMessageBodySize);
- keyBuilder = new StringBuilder();
- }
+ void shutdown();
- public MessageExtEncoder getEncoder() {
- return encoder;
+ void wakeUpFlush();
+
+ void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt);
+
+ CompletableFuture<PutMessageStatus> handleDiskFlush(AppendMessageResult result, MessageExt messageExt);
+ }
+
+ class DefaultFlushManager implements FlushManager {
+
+ private final FlushCommitLogService flushCommitLogService;
+
+ //If TransientStorePool enabled, we must flush message to FileChannel at fixed periods
+ private final FlushCommitLogService commitLogService;
+
+ public DefaultFlushManager() {
+ if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
+ this.flushCommitLogService = new CommitLog.GroupCommitService();
+ } else {
+ this.flushCommitLogService = new CommitLog.FlushRealTimeService();
+ }
+
+ this.commitLogService = new CommitLog.CommitRealTimeService();
}
- public StringBuilder getKeyBuilder() {
- return keyBuilder;
+ @Override
+ public void start() {
+ this.flushCommitLogService.start();
+
+ if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
+ this.commitLogService.start();
+ }
}
- }
- static class PutMessageContext {
- private String topicQueueTableKey;
- private long[] phyPos;
- private int batchSize;
+ public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult,
+ MessageExt messageExt) {
+ // Synchronization flush
+ if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
+ final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
+ if (messageExt.isWaitStoreMsgOK()) {
+ GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(), CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
+ service.putRequest(request);
+ CompletableFuture<PutMessageStatus> flushOkFuture = request.future();
+ PutMessageStatus flushStatus = null;
+ try {
+ flushStatus = flushOkFuture.get(CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout(),
+ TimeUnit.MILLISECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ //flushOK=false;
+ }
+ if (flushStatus != PutMessageStatus.PUT_OK) {
+ log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()
+ + " client address: " + messageExt.getBornHostString());
+ putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);
+ }
+ } else {
+ service.wakeup();
+ }
+ }
+ // Asynchronous flush
+ else {
+ if (!CommitLog.this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
+ flushCommitLogService.wakeup();
+ } else {
+ commitLogService.wakeup();
+ }
+ }
+ }
- public PutMessageContext(String topicQueueTableKey) {
- this.topicQueueTableKey = topicQueueTableKey;
+ @Override
+ public CompletableFuture<PutMessageStatus> handleDiskFlush(AppendMessageResult result, MessageExt messageExt) {
+ // Synchronization flush
+ if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
+ final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
+ if (messageExt.isWaitStoreMsgOK()) {
+ GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(), CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
+ flushDiskWatcher.add(request);
+ service.putRequest(request);
+ return request.future();
+ } else {
+ service.wakeup();
+ return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);
+ }
+ }
+ // Asynchronous flush
+ else {
+ if (!CommitLog.this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
+ flushCommitLogService.wakeup();
+ } else {
+ commitLogService.wakeup();
+ }
+ return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);
+ }
}
- public String getTopicQueueTableKey() {
- return topicQueueTableKey;
+ @Override
+ public void wakeUpFlush() {
+ // now wake up flush thread.
+ flushCommitLogService.wakeup();
}
- public long[] getPhyPos() {
- return phyPos;
+ @Override
+ public void shutdown() {
+ if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
+ this.commitLogService.shutdown();
+ }
+
+ this.flushCommitLogService.shutdown();
}
- public void setPhyPos(long[] phyPos) {
- this.phyPos = phyPos;
+ }
+
+ public int getCommitLogSize() {
+ return commitLogSize;
+ }
+
+ public MappedFileQueue getMappedFileQueue() {
+ return mappedFileQueue;
+ }
+
+ public MessageStore getMessageStore() {
+ return defaultMessageStore;
+ }
+
+ @Override
+ public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) {
+ this.getMappedFileQueue().swapMap(reserveNum, forceSwapIntervalMs, normalSwapIntervalMs);
+ }
+
+ public boolean isMappedFilesEmpty() {
+ return this.mappedFileQueue.isMappedFilesEmpty();
+ }
+
+ @Override
+ public void cleanSwappedMap(long forceCleanSwapIntervalMs) {
+ this.getMappedFileQueue().cleanSwappedMap(forceCleanSwapIntervalMs);
+ }
+
+ static class PutMessageThreadLocal {
+ private MessageExtEncoder encoder;
+ private StringBuilder keyBuilder;
+
- PutMessageThreadLocal(int size) {
- encoder = new MessageExtEncoder(size);
++ PutMessageThreadLocal(int maxMessageBodySize) {
++ encoder = new MessageExtEncoder(maxMessageBodySize);
+ keyBuilder = new StringBuilder();
}
- public int getBatchSize() {
- return batchSize;
+ public MessageExtEncoder getEncoder() {
+ return encoder;
}
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
+ public StringBuilder getKeyBuilder() {
+ return keyBuilder;
}
}
}
diff --cc store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
index 8cbe690ea,0da12a232..1cb60b87d
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
@@@ -16,7 -16,7 +16,8 @@@
*/
package org.apache.rocketmq.store;
+ import com.google.common.hash.Hashing;
+import io.openmessaging.storage.dledger.entry.DLedgerEntry;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
@@@ -305,6 -346,60 +347,60 @@@ public class DefaultMessageStore implem
this.shutdown = false;
}
+ private void doRecheckReputOffsetFromCq() throws InterruptedException {
+ if (!messageStoreConfig.isRecheckReputOffsetFromCq()) {
+ return;
+ }
+
+ /**
+ * 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog;
+ * 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go;
+ * 3. Calculate the reput offset according to the consume queue;
+ * 4. Make sure the fall-behind messages to be dispatched before starting the commitlog, especially when the broker role are automatically changed.
+ */
+ long maxPhysicalPosInLogicQueue = commitLog.getMinOffset();
+ for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.getConsumeQueueTable().values()) {
+ for (ConsumeQueueInterface logic : maps.values()) {
+ if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) {
+ maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset();
+ }
+ }
+ }
+ // If maxPhyPos(CQs) < minPhyPos(CommitLog), some newly deleted topics may be re-dispatched into cqs mistakenly.
+ if (maxPhysicalPosInLogicQueue < 0) {
+ maxPhysicalPosInLogicQueue = 0;
+ }
+ if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) {
+ maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset();
+ /**
+ * This happens in following conditions:
+ * 1. If someone removes all the consumequeue files or the disk get damaged.
+ * 2. Launch a new broker, and copy the commitlog from other brokers.
+ *
+ * All the conditions has the same in common that the maxPhysicalPosInLogicQueue should be 0.
+ * If the maxPhysicalPosInLogicQueue is gt 0, there maybe something wrong.
+ */
+ LOGGER.warn("[TooSmallCqOffset] maxPhysicalPosInLogicQueue={} clMinOffset={}", maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset());
+ }
+ LOGGER.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMinOffset={} clMaxOffset={} clConfirmedOffset={}",
- maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset());
++ maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset());
+ this.reputMessageService.setReputFromOffset(maxPhysicalPosInLogicQueue);
+
+ /**
+ * 1. Finish dispatching the messages fall behind, then to start other services.
+ * 2. DLedger committedPos may be missing, so here just require dispatchBehindBytes <= 0
+ */
+ while (true) {
+ if (dispatchBehindBytes() <= 0) {
+ break;
+ }
+ Thread.sleep(1000);
+ LOGGER.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes());
+ }
+ this.recoverTopicQueueTable();
+ }
+
+ @Override
public void shutdown() {
if (!this.shutdown) {
this.shutdown = true;
@@@ -446,21 -481,27 +482,27 @@@
@Override
public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {
- PutMessageStatus checkStoreStatus = this.checkStoreStatus();
- if (checkStoreStatus != PutMessageStatus.PUT_OK) {
- return CompletableFuture.completedFuture(new PutMessageResult(checkStoreStatus, null));
- }
- PutMessageStatus msgCheckStatus = this.checkMessage(msg);
- if (msgCheckStatus == PutMessageStatus.MESSAGE_ILLEGAL) {
- return CompletableFuture.completedFuture(new PutMessageResult(msgCheckStatus, null));
+ for (PutMessageHook putMessageHook : putMessageHookList) {
+ PutMessageResult handleResult = putMessageHook.executeBeforePutMessage(msg);
+ if (handleResult != null) {
+ return CompletableFuture.completedFuture(handleResult);
+ }
}
- PutMessageStatus lmqMsgCheckStatus = this.checkLmqMessage(msg);
- if (msgCheckStatus == PutMessageStatus.LMQ_CONSUME_QUEUE_NUM_EXCEEDED) {
- return CompletableFuture.completedFuture(new PutMessageResult(lmqMsgCheckStatus, null));
+ if (msg.getProperties().containsKey(MessageConst.PROPERTY_INNER_NUM)
- && !MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
++ && !MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
+ LOGGER.warn("[BUG]The message had property {} but is not an inner batch", MessageConst.PROPERTY_INNER_NUM);
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));
}
+ if (MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
+ Optional<TopicConfig> topicConfig = this.getTopicConfig(msg.getTopic());
+ if (!QueueTypeUtils.isBatchCq(topicConfig)) {
+ LOGGER.error("[BUG]The message is an inner batch but cq type is not batch cq");
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));
+ }
+ }
long beginTime = this.getSystemClock().now();
CompletableFuture<PutMessageResult> putResultFuture = this.commitLog.asyncPutMessage(msg);
@@@ -522,15 -563,15 +564,15 @@@
private PutMessageResult waitForPutResult(CompletableFuture<PutMessageResult> putMessageResultFuture) {
try {
int putMessageTimeout =
-- Math.max(this.messageStoreConfig.getSyncFlushTimeout(),
-- this.messageStoreConfig.getSlaveTimeout()) + 5000;
++ Math.max(this.messageStoreConfig.getSyncFlushTimeout(),
++ this.messageStoreConfig.getSlaveTimeout()) + 5000;
return putMessageResultFuture.get(putMessageTimeout, TimeUnit.MILLISECONDS);
} catch (ExecutionException | InterruptedException e) {
return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);
} catch (TimeoutException e) {
- log.error("usually it will never timeout, putMessageTimeout is much bigger than slaveTimeout and "
- + "flushTimeout so the result can be got anyway, but in some situations timeout will happen like full gc "
- + "process hangs or other unexpected situations.");
+ LOGGER.error("usually it will never timeout, putMessageTimeout is much bigger than slaveTimeout and "
- + "flushTimeout so the result can be got anyway, but in some situations timeout will happen like full gc "
- + "process hangs or other unexpected situations.");
++ + "flushTimeout so the result can be got anyway, but in some situations timeout will happen like full gc "
++ + "process hangs or other unexpected situations.");
return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);
}
}
@@@ -995,15 -1142,29 +1144,15 @@@
lastQueryMsgTime = msg.getStoreTimestamp();
}
-// String[] keyArray = msg.getKeys().split(MessageConst.KEY_SEPARATOR);
-// if (topic.equals(msg.getTopic())) {
-// for (String k : keyArray) {
-// if (k.equals(key)) {
-// match = true;
-// break;
-// }
-// }
-// }
-
- if (match) {
- SelectMappedBufferResult result = this.commitLog.getData(offset, false);
- if (result != null) {
- int size = result.getByteBuffer().getInt(0);
- result.getByteBuffer().limit(size);
- result.setSize(size);
- queryMessageResult.addMessage(result);
- }
- } else {
- LOGGER.warn("queryMessage hash duplicate, topic={}, key={}", topic, key);
+ SelectMappedBufferResult result = this.commitLog.getData(offset, false);
+ if (result != null) {
+ int size = result.getByteBuffer().getInt(0);
+ result.getByteBuffer().limit(size);
+ result.setSize(size);
+ queryMessageResult.addMessage(result);
}
} catch (Exception e) {
- log.error("queryMessage exception", e);
+ LOGGER.error("queryMessage exception", e);
}
}
@@@ -1139,9 -1293,10 +1281,10 @@@
int msgIdLength = (inetSocketAddress.getAddress() instanceof Inet6Address) ? 16 + 4 + 8 : 4 + 4 + 8;
final ByteBuffer msgIdMemory = ByteBuffer.allocate(msgIdLength);
String msgId =
- MessageDecoder.createMessageId(msgIdMemory, MessageExt.socketAddress2ByteBuffer(storeHost), offsetPy);
+ MessageDecoder.createMessageId(msgIdMemory, MessageExt.socketAddress2ByteBuffer(storeHost), offsetPy);
- messageIds.put(msgId, nextOffset++);
- if (nextOffset > maxOffset) {
+ messageIds.put(msgId, cqUnit.getQueueOffset());
+ nextOffset = cqUnit.getQueueOffset() + cqUnit.getBatchNum();
+ if (nextOffset >= maxOffset) {
return messageIds;
}
}
diff --cc store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
index 46797b343,1a6d6ae48..5f1ee6989
--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
+++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
@@@ -85,9 -89,11 +93,11 @@@ public class MessageStoreConfig
// The number of hours to keep a log file before deleting it (in hours)
@ImportantField
private int fileReservedTime = 72;
+ @ImportantField
+ private int deleteFileBatchMax = 10;
// Flow control for ConsumeQueue
private int putMsgIndexHightWater = 600000;
- // The maximum size of message,default is 4M
+ // The maximum size of message body,default is 4M,4M only for body length,not include others.
private int maxMessageSize = 1024 * 1024 * 4;
// Whether check the CRC32 of the records consumed.
// This ensures no on-the-wire or on-disk corruption to the messages occurred.
diff --cc store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
index 5474fa46b,42064a35a..ac7d31fa3
--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
@@@ -55,8 -50,7 +50,8 @@@ import org.apache.rocketmq.store.PutMes
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.StoreStatsService;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
- import org.apache.rocketmq.store.schedule.ScheduleMessageService;
+ import org.apache.rocketmq.store.logfile.MappedFile;
/**
* Store all metadata downtime for recovery, data protection reliability
diff --cc store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAConnection.java
index 5d88f00f2,3b7241065..aa5da1eb4
--- a/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAConnection.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAConnection.java
@@@ -26,24 -27,24 +27,25 @@@ import org.apache.rocketmq.common.const
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.apache.rocketmq.remoting.netty.NettySystemConfig;
import org.apache.rocketmq.store.SelectMappedBufferResult;
- public class HAConnection {
+ public class DefaultHAConnection implements HAConnection {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
- private final HAService haService;
+ private final DefaultHAService haService;
private final SocketChannel socketChannel;
- private final String clientAddr;
- private final WriteSocketService writeSocketService;
- private final ReadSocketService readSocketService;
-
+ private final String clientAddress;
+ private WriteSocketService writeSocketService;
+ private ReadSocketService readSocketService;
+ private volatile HAConnectionState currentState = HAConnectionState.TRANSFER;
private volatile long slaveRequestOffset = -1;
private volatile long slaveAckOffset = -1;
+ private FlowMonitor flowMonitor;
- public HAConnection(final HAService haService, final SocketChannel socketChannel) throws IOException {
+ public DefaultHAConnection(final DefaultHAService haService, final SocketChannel socketChannel) throws IOException {
this.haService = haService;
this.socketChannel = socketChannel;
- this.clientAddr = this.socketChannel.socket().getRemoteSocketAddress().toString();
+ this.clientAddress = this.socketChannel.socket().getRemoteSocketAddress().toString();
this.socketChannel.configureBlocking(false);
this.socketChannel.socket().setSoLinger(false, -1);
this.socketChannel.socket().setTcpNoDelay(true);
@@@ -382,6 -431,9 +436,9 @@@
@Override
public String getServiceName() {
+ if (haService.getDefaultMessageStore().getBrokerConfig().isInBrokerContainer()) {
- return haService.getDefaultMessageStore().getBrokerConfig().getLoggerIdentifier() + WriteSocketService.class.getSimpleName();
++ return haService.getDefaultMessageStore().getBrokerIdentity().getLoggerIdentifier() + WriteSocketService.class.getSimpleName();
+ }
return WriteSocketService.class.getSimpleName();
}
diff --cc store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
index bdea3307b,8b59f547e..1684eed12
--- a/store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
+++ b/store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
@@@ -19,7 -19,10 +19,8 @@@ package org.apache.rocketmq.store.index
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
import java.util.List;
+ import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
@@@ -32,15 -36,18 +34,16 @@@ public class IndexFile
private static int invalidIndex = 0;
private final int hashSlotNum;
private final int indexNum;
+ private final int fileTotalSize;
private final MappedFile mappedFile;
- private final FileChannel fileChannel;
private final MappedByteBuffer mappedByteBuffer;
private final IndexHeader indexHeader;
public IndexFile(final String fileName, final int hashSlotNum, final int indexNum,
final long endPhyOffset, final long endTimestamp) throws IOException {
- int fileTotalSize =
+ this.fileTotalSize =
IndexHeader.INDEX_HEADER_SIZE + (hashSlotNum * hashSlotSize) + (indexNum * indexSize);
- this.mappedFile = new MappedFile(fileName, fileTotalSize);
+ this.mappedFile = new DefaultMappedFile(fileName, fileTotalSize);
- this.fileChannel = this.mappedFile.getFileChannel();
this.mappedByteBuffer = this.mappedFile.getMappedByteBuffer();
this.hashSlotNum = hashSlotNum;
this.indexNum = indexNum;
@@@ -67,6 -78,17 +74,11 @@@
this.indexHeader.load();
}
+ public void shutdown() {
+ this.flush();
+ UtilAll.cleanBuffer(this.mappedByteBuffer);
-
- try {
- this.fileChannel.close();
- } catch (IOException e) {
- log.error("Shutdown error in index file", e);
- }
+ }
+
public void flush() {
long beginTime = System.currentTimeMillis();
if (this.mappedFile.hold()) {
diff --cc store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
index b1fcbf20b,095bb261f..89709989f
--- a/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
+++ b/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
@@@ -377,12 -376,13 +376,13 @@@ public class DefaultMappedFile extends
}
if (commitLeastPages > 0) {
- return ((write / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE)) >= commitLeastPages;
+ return ((write / OS_PAGE_SIZE) - (commit / OS_PAGE_SIZE)) >= commitLeastPages;
}
- return write > flush;
+ return write > commit;
}
+ @Override
public int getFlushedPosition() {
return flushedPosition.get();
}
diff --cc store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
index 605552bd0,1ea987ada..7ec9477d7
--- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
+++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
@@@ -19,9 -19,19 +19,20 @@@ package org.apache.rocketmq.store.stats
import java.util.HashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
+ import org.apache.commons.lang3.tuple.Pair;
+ import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.ThreadFactoryImpl;
+ import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
+ import org.apache.rocketmq.common.statistics.StatisticsItem;
+ import org.apache.rocketmq.common.statistics.StatisticsItemFormatter;
+ import org.apache.rocketmq.common.statistics.StatisticsItemPrinter;
+ import org.apache.rocketmq.common.statistics.StatisticsItemScheduledIncrementPrinter;
+ import org.apache.rocketmq.common.statistics.StatisticsItemScheduledPrinter;
+ import org.apache.rocketmq.common.statistics.StatisticsItemStateGetter;
+ import org.apache.rocketmq.common.statistics.StatisticsKindMeta;
+ import org.apache.rocketmq.common.statistics.StatisticsManager;
+import org.apache.rocketmq.common.stats.Stats;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.stats.MomentStatsItemSet;
@@@ -30,38 -40,81 +41,85 @@@ import org.apache.rocketmq.common.stats
public class BrokerStatsManager {
- public static final String QUEUE_PUT_NUMS = "QUEUE_PUT_NUMS";
- public static final String QUEUE_PUT_SIZE = "QUEUE_PUT_SIZE";
- public static final String QUEUE_GET_NUMS = "QUEUE_GET_NUMS";
- public static final String QUEUE_GET_SIZE = "QUEUE_GET_SIZE";
- public static final String TOPIC_PUT_NUMS = "TOPIC_PUT_NUMS";
- public static final String TOPIC_PUT_SIZE = "TOPIC_PUT_SIZE";
+ @Deprecated public static final String QUEUE_PUT_NUMS = Stats.QUEUE_PUT_NUMS;
+ @Deprecated public static final String QUEUE_PUT_SIZE = Stats.QUEUE_PUT_SIZE;
+ @Deprecated public static final String QUEUE_GET_NUMS = Stats.QUEUE_GET_NUMS;
+ @Deprecated public static final String QUEUE_GET_SIZE = Stats.QUEUE_GET_SIZE;
+ @Deprecated public static final String TOPIC_PUT_NUMS = Stats.TOPIC_PUT_NUMS;
+ @Deprecated public static final String TOPIC_PUT_SIZE = Stats.TOPIC_PUT_SIZE;
++
+ @Deprecated public static final String GROUP_GET_NUMS = Stats.GROUP_GET_NUMS;
+ @Deprecated public static final String GROUP_GET_SIZE = Stats.GROUP_GET_SIZE;
++
+ @Deprecated public static final String SNDBCK_PUT_NUMS = Stats.SNDBCK_PUT_NUMS;
+ @Deprecated public static final String BROKER_PUT_NUMS = Stats.BROKER_PUT_NUMS;
+ @Deprecated public static final String BROKER_GET_NUMS = Stats.BROKER_GET_NUMS;
+ @Deprecated public static final String GROUP_GET_FROM_DISK_NUMS = Stats.GROUP_GET_FROM_DISK_NUMS;
+ @Deprecated public static final String GROUP_GET_FROM_DISK_SIZE = Stats.GROUP_GET_FROM_DISK_SIZE;
+ @Deprecated public static final String BROKER_GET_FROM_DISK_NUMS = Stats.BROKER_GET_FROM_DISK_NUMS;
+ @Deprecated public static final String BROKER_GET_FROM_DISK_SIZE = Stats.BROKER_GET_FROM_DISK_SIZE;
+ // For commercial
+ @Deprecated public static final String COMMERCIAL_SEND_TIMES = Stats.COMMERCIAL_SEND_TIMES;
+ @Deprecated public static final String COMMERCIAL_SNDBCK_TIMES = Stats.COMMERCIAL_SNDBCK_TIMES;
+ @Deprecated public static final String COMMERCIAL_RCV_TIMES = Stats.COMMERCIAL_RCV_TIMES;
+ @Deprecated public static final String COMMERCIAL_RCV_EPOLLS = Stats.COMMERCIAL_RCV_EPOLLS;
+ @Deprecated public static final String COMMERCIAL_SEND_SIZE = Stats.COMMERCIAL_SEND_SIZE;
+ @Deprecated public static final String COMMERCIAL_RCV_SIZE = Stats.COMMERCIAL_RCV_SIZE;
+ @Deprecated public static final String COMMERCIAL_PERM_FAILURES = Stats.COMMERCIAL_PERM_FAILURES;
++
++
+ // Send message latency
+ public static final String TOPIC_PUT_LATENCY = "TOPIC_PUT_LATENCY";
- public static final String GROUP_GET_NUMS = "GROUP_GET_NUMS";
- public static final String GROUP_GET_SIZE = "GROUP_GET_SIZE";
+ public static final String GROUP_ACK_NUMS = "GROUP_ACK_NUMS";
+ public static final String GROUP_CK_NUMS = "GROUP_CK_NUMS";
- public static final String SNDBCK_PUT_NUMS = "SNDBCK_PUT_NUMS";
+ public static final String DLQ_PUT_NUMS = "DLQ_PUT_NUMS";
- public static final String BROKER_PUT_NUMS = "BROKER_PUT_NUMS";
- public static final String BROKER_GET_NUMS = "BROKER_GET_NUMS";
+ public static final String BROKER_ACK_NUMS = "BROKER_ACK_NUMS";
+ public static final String BROKER_CK_NUMS = "BROKER_CK_NUMS";
- public static final String GROUP_GET_FROM_DISK_NUMS = "GROUP_GET_FROM_DISK_NUMS";
- public static final String GROUP_GET_FROM_DISK_SIZE = "GROUP_GET_FROM_DISK_SIZE";
- public static final String BROKER_GET_FROM_DISK_NUMS = "BROKER_GET_FROM_DISK_NUMS";
- public static final String BROKER_GET_FROM_DISK_SIZE = "BROKER_GET_FROM_DISK_SIZE";
-
+ public static final String SNDBCK2DLQ_TIMES = "SNDBCK2DLQ_TIMES";
+
- // For commercial
- public static final String COMMERCIAL_SEND_TIMES = "COMMERCIAL_SEND_TIMES";
- public static final String COMMERCIAL_SNDBCK_TIMES = "COMMERCIAL_SNDBCK_TIMES";
- public static final String COMMERCIAL_RCV_TIMES = "COMMERCIAL_RCV_TIMES";
- public static final String COMMERCIAL_RCV_EPOLLS = "COMMERCIAL_RCV_EPOLLS";
- public static final String COMMERCIAL_SEND_SIZE = "COMMERCIAL_SEND_SIZE";
- public static final String COMMERCIAL_RCV_SIZE = "COMMERCIAL_RCV_SIZE";
- public static final String COMMERCIAL_PERM_FAILURES = "COMMERCIAL_PERM_FAILURES";
public static final String COMMERCIAL_OWNER = "Owner";
- // Message Size limit for one api-calling count.
- public static final double SIZE_PER_COUNT = 64 * 1024;
+
+ public static final String ACCOUNT_OWNER_PARENT = "OWNER_PARENT";
+ public static final String ACCOUNT_OWNER_SELF = "OWNER_SELF";
+
+ public static final long ACCOUNT_STAT_INVERTAL = 60 * 1000;
+ public static final String ACCOUNT_AUTH_TYPE = "AUTH_TYPE";
+
+ public static final String ACCOUNT_SEND = "SEND";
+ public static final String ACCOUNT_RCV = "RCV";
+ public static final String ACCOUNT_SEND_BACK = "SEND_BACK";
+ public static final String ACCOUNT_SEND_BACK_TO_DLQ = "SEND_BACK_TO_DLQ";
+ public static final String ACCOUNT_AUTH_FAILED = "AUTH_FAILED";
+ public static final String ACCOUNT_SEND_REJ = "SEND_REJ";
+ public static final String ACCOUNT_REV_REJ = "RCV_REJ";
+
+ public static final String MSG_NUM = "MSG_NUM";
+ public static final String MSG_SIZE = "MSG_SIZE";
+ public static final String SUCCESS_MSG_NUM = "SUCCESS_MSG_NUM";
+ public static final String FAILURE_MSG_NUM = "FAILURE_MSG_NUM";
+ public static final String COMMERCIAL_MSG_NUM = "COMMERCIAL_MSG_NUM";
+ public static final String SUCCESS_REQ_NUM = "SUCCESS_REQ_NUM";
+ public static final String FAILURE_REQ_NUM = "FAILURE_REQ_NUM";
+ public static final String SUCCESS_MSG_SIZE = "SUCCESS_MSG_SIZE";
+ public static final String FAILURE_MSG_SIZE = "FAILURE_MSG_SIZE";
+ public static final String RT = "RT";
+ public static final String INNER_RT = "INNER_RT";
- public static final String GROUP_GET_FALL_SIZE = "GROUP_GET_FALL_SIZE";
- public static final String GROUP_GET_FALL_TIME = "GROUP_GET_FALL_TIME";
+ @Deprecated public static final String GROUP_GET_FALL_SIZE = Stats.GROUP_GET_FALL_SIZE;
+ @Deprecated public static final String GROUP_GET_FALL_TIME = Stats.GROUP_GET_FALL_TIME;
// Pull Message Latency
- public static final String GROUP_GET_LATENCY = "GROUP_GET_LATENCY";
+ @Deprecated public static final String GROUP_GET_LATENCY = Stats.GROUP_GET_LATENCY;
+
+ // Consumer Register Time
+ public static final String CONSUMER_REGISTER_TIME = "CONSUMER_REGISTER_TIME";
+ // Producer Register Time
+ public static final String PRODUCER_REGISTER_TIME = "PRODUCER_REGISTER_TIME";
+ public static final String CHANNEL_ACTIVITY = "CHANNEL_ACTIVITY";
+ public static final String CHANNEL_ACTIVITY_CONNECT = "CONNECT";
+ public static final String CHANNEL_ACTIVITY_IDLE = "IDLE";
+ public static final String CHANNEL_ACTIVITY_EXCEPTION = "EXCEPTION";
+ public static final String CHANNEL_ACTIVITY_CLOSE = "CLOSE";
+
/**
* read disk follow stats
*/
@@@ -80,33 -151,130 +156,130 @@@
public BrokerStatsManager(String clusterName, boolean enableQueueStat) {
this.clusterName = clusterName;
this.enableQueueStat = enableQueueStat;
+ initScheduleService();
+ init();
+ }
+
+ public void init() {
+ momentStatsItemSetFallSize = new MomentStatsItemSet(GROUP_GET_FALL_SIZE,
+ scheduledExecutorService, log);
+
+ momentStatsItemSetFallTime = new MomentStatsItemSet(GROUP_GET_FALL_TIME,
+ scheduledExecutorService, log);
if (enableQueueStat) {
- this.statsTable.put(QUEUE_PUT_NUMS, new StatsItemSet(QUEUE_PUT_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(QUEUE_PUT_SIZE, new StatsItemSet(QUEUE_PUT_SIZE, this.scheduledExecutorService, log));
- this.statsTable.put(QUEUE_GET_NUMS, new StatsItemSet(QUEUE_GET_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(QUEUE_GET_SIZE, new StatsItemSet(QUEUE_GET_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.QUEUE_PUT_NUMS, new StatsItemSet(Stats.QUEUE_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.QUEUE_PUT_SIZE, new StatsItemSet(Stats.QUEUE_PUT_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.QUEUE_GET_NUMS, new StatsItemSet(Stats.QUEUE_GET_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.QUEUE_GET_SIZE, new StatsItemSet(Stats.QUEUE_GET_SIZE, this.scheduledExecutorService, log));
}
- this.statsTable.put(TOPIC_PUT_NUMS, new StatsItemSet(TOPIC_PUT_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(TOPIC_PUT_SIZE, new StatsItemSet(TOPIC_PUT_SIZE, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_NUMS, new StatsItemSet(GROUP_GET_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_SIZE, new StatsItemSet(GROUP_GET_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.TOPIC_PUT_NUMS, new StatsItemSet(Stats.TOPIC_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.TOPIC_PUT_SIZE, new StatsItemSet(Stats.TOPIC_PUT_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.GROUP_GET_NUMS, new StatsItemSet(Stats.GROUP_GET_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.GROUP_GET_SIZE, new StatsItemSet(Stats.GROUP_GET_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(GROUP_ACK_NUMS, new StatsItemSet(GROUP_ACK_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(GROUP_CK_NUMS, new StatsItemSet(GROUP_CK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_LATENCY, new StatsItemSet(GROUP_GET_LATENCY, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.GROUP_GET_LATENCY, new StatsItemSet(Stats.GROUP_GET_LATENCY, this.scheduledExecutorService, log));
+ this.statsTable.put(TOPIC_PUT_LATENCY, new StatsItemSet(TOPIC_PUT_LATENCY, this.scheduledExecutorService, log));
- this.statsTable.put(SNDBCK_PUT_NUMS, new StatsItemSet(SNDBCK_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.SNDBCK_PUT_NUMS, new StatsItemSet(Stats.SNDBCK_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(DLQ_PUT_NUMS, new StatsItemSet(DLQ_PUT_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(BROKER_PUT_NUMS, new StatsItemSet(BROKER_PUT_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(BROKER_GET_NUMS, new StatsItemSet(BROKER_GET_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.BROKER_PUT_NUMS, new StatsItemSet(Stats.BROKER_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.BROKER_GET_NUMS, new StatsItemSet(Stats.BROKER_GET_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(Stats.GROUP_GET_FROM_DISK_NUMS, new StatsItemSet(Stats.GROUP_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(Stats.GROUP_GET_FROM_DISK_SIZE, new StatsItemSet(Stats.GROUP_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
- this.statsTable.put(Stats.BROKER_GET_FROM_DISK_NUMS, new StatsItemSet(Stats.BROKER_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(Stats.BROKER_GET_FROM_DISK_SIZE, new StatsItemSet(Stats.BROKER_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
-
- this.statsTable.put(Stats.COMMERCIAL_SEND_TIMES, new StatsItemSet(Stats.COMMERCIAL_SEND_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_RCV_TIMES, new StatsItemSet(Stats.COMMERCIAL_RCV_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_SEND_SIZE, new StatsItemSet(Stats.COMMERCIAL_SEND_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_RCV_SIZE, new StatsItemSet(Stats.COMMERCIAL_RCV_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_RCV_EPOLLS, new StatsItemSet(Stats.COMMERCIAL_RCV_EPOLLS, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_SNDBCK_TIMES, new StatsItemSet(Stats.COMMERCIAL_SNDBCK_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_PERM_FAILURES, new StatsItemSet(Stats.COMMERCIAL_PERM_FAILURES, this.commercialExecutor, COMMERCIAL_LOG));
+ this.statsTable.put(BROKER_ACK_NUMS, new StatsItemSet(BROKER_ACK_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(BROKER_CK_NUMS, new StatsItemSet(BROKER_CK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_FROM_DISK_NUMS,
- new StatsItemSet(GROUP_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_FROM_DISK_SIZE,
- new StatsItemSet(GROUP_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
- this.statsTable.put(BROKER_GET_FROM_DISK_NUMS,
- new StatsItemSet(BROKER_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(BROKER_GET_FROM_DISK_SIZE,
- new StatsItemSet(BROKER_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
++ this.statsTable.put(Stats.GROUP_GET_FROM_DISK_NUMS,
++ new StatsItemSet(Stats.GROUP_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
++ this.statsTable.put(Stats.GROUP_GET_FROM_DISK_SIZE,
++ new StatsItemSet(Stats.GROUP_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
++ this.statsTable.put(Stats.BROKER_GET_FROM_DISK_NUMS,
++ new StatsItemSet(Stats.BROKER_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
++ this.statsTable.put(Stats.BROKER_GET_FROM_DISK_SIZE,
++ new StatsItemSet(Stats.BROKER_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
+
+ this.statsTable.put(SNDBCK2DLQ_TIMES,
+ new StatsItemSet(SNDBCK2DLQ_TIMES, this.scheduledExecutorService, DLQ_STAT_LOG));
+
- this.statsTable.put(COMMERCIAL_SEND_TIMES,
- new StatsItemSet(COMMERCIAL_SEND_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_RCV_TIMES,
- new StatsItemSet(COMMERCIAL_RCV_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_SEND_SIZE,
- new StatsItemSet(COMMERCIAL_SEND_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_RCV_SIZE,
- new StatsItemSet(COMMERCIAL_RCV_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_RCV_EPOLLS,
- new StatsItemSet(COMMERCIAL_RCV_EPOLLS, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_SNDBCK_TIMES,
- new StatsItemSet(COMMERCIAL_SNDBCK_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_PERM_FAILURES,
- new StatsItemSet(COMMERCIAL_PERM_FAILURES, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_SEND_TIMES,
++ new StatsItemSet(Stats.COMMERCIAL_SEND_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_RCV_TIMES,
++ new StatsItemSet(Stats.COMMERCIAL_RCV_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_SEND_SIZE,
++ new StatsItemSet(Stats.COMMERCIAL_SEND_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_RCV_SIZE,
++ new StatsItemSet(Stats.COMMERCIAL_RCV_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_RCV_EPOLLS,
++ new StatsItemSet(Stats.COMMERCIAL_RCV_EPOLLS, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_SNDBCK_TIMES,
++ new StatsItemSet(Stats.COMMERCIAL_SNDBCK_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_PERM_FAILURES,
++ new StatsItemSet(Stats.COMMERCIAL_PERM_FAILURES, this.commercialExecutor, COMMERCIAL_LOG));
+
+ this.statsTable.put(CONSUMER_REGISTER_TIME,
+ new StatsItemSet(CONSUMER_REGISTER_TIME, this.scheduledExecutorService, log));
+ this.statsTable.put(PRODUCER_REGISTER_TIME,
+ new StatsItemSet(PRODUCER_REGISTER_TIME, this.scheduledExecutorService, log));
+
+ this.statsTable.put(CHANNEL_ACTIVITY, new StatsItemSet(CHANNEL_ACTIVITY, this.scheduledExecutorService, log));
+
+ StatisticsItemFormatter formatter = new StatisticsItemFormatter();
+ accountStatManager.setBriefMeta(new Pair[] {
+ Pair.of(RT, new long[][] {{50, 50}, {100, 10}, {1000, 10}}),
+ Pair.of(INNER_RT, new long[][] {{10, 10}, {100, 10}, {1000, 10}})});
+ String[] itemNames = new String[] {
+ MSG_NUM, SUCCESS_MSG_NUM, FAILURE_MSG_NUM, COMMERCIAL_MSG_NUM,
+ SUCCESS_REQ_NUM, FAILURE_REQ_NUM,
+ MSG_SIZE, SUCCESS_MSG_SIZE, FAILURE_MSG_SIZE,
+ RT, INNER_RT};
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_SEND, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_RCV, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_SEND_BACK, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_SEND_BACK_TO_DLQ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_SEND_REJ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_REV_REJ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.setStatisticsItemStateGetter(new StatisticsItemStateGetter() {
+ @Override
+ public boolean online(StatisticsItem item) {
+ String[] strArr = null;
+ try {
+ strArr = splitAccountStatKey(item.getStatObject());
+ } catch (Exception e) {
+ log.warn("parse account stat key failed, key: {}", item.getStatObject());
+ return false;
+ }
+
+ // TODO ugly
+ if (strArr == null || strArr.length < 4) {
+ return false;
+ }
+
+ String instanceId = strArr[1];
+ String topic = strArr[2];
+ String group = strArr[3];
+
+ String kind = item.getStatKind();
+ if (ACCOUNT_SEND.equals(kind) || ACCOUNT_SEND_REJ.equals(kind)) {
+ return produerStateGetter.online(instanceId, group, topic);
+ } else if (ACCOUNT_RCV.equals(kind) || ACCOUNT_SEND_BACK.equals(kind) || ACCOUNT_SEND_BACK_TO_DLQ.equals(kind) || ACCOUNT_REV_REJ.equals(kind)) {
+ return consumerStateGetter.online(instanceId, group, topic);
+ }
+ return false;
+ }
+ });
+ }
+
+ private void initScheduleService() {
+ this.scheduledExecutorService =
+ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig));
+ this.commercialExecutor =
+ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig));
+ this.accountExecutor =
+ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig));
}
public MomentStatsItemSet getMomentStatsItemSetFallSize() {
@@@ -196,8 -380,32 +385,32 @@@
}
}
+ public void incConsumerRegisterTime(final int incValue) {
+ this.statsTable.get(CONSUMER_REGISTER_TIME).addValue(this.clusterName, incValue, 1);
+ }
+
+ public void incProducerRegisterTime(final int incValue) {
+ this.statsTable.get(PRODUCER_REGISTER_TIME).addValue(this.clusterName, incValue, 1);
+ }
+
+ public void incChannelConnectNum() {
+ this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_CONNECT, 1, 1);
+ }
+
+ public void incChannelCloseNum() {
+ this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_CLOSE, 1, 1);
+ }
+
+ public void incChannelExceptionNum() {
+ this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_EXCEPTION, 1, 1);
+ }
+
+ public void incChannelIdleNum() {
+ this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_IDLE, 1, 1);
+ }
+
public void incTopicPutNums(final String topic) {
- this.statsTable.get(TOPIC_PUT_NUMS).addValue(topic, 1, 1);
+ this.statsTable.get(Stats.TOPIC_PUT_NUMS).addValue(topic, 1, 1);
}
public void incTopicPutNums(final String topic, int num, int times) {
@@@ -210,9 -418,19 +423,19 @@@
public void incGroupGetNums(final String group, final String topic, final int incValue) {
final String statsKey = buildStatsKey(topic, group);
- this.statsTable.get(GROUP_GET_NUMS).addValue(statsKey, incValue, 1);
+ this.statsTable.get(Stats.GROUP_GET_NUMS).addValue(statsKey, incValue, 1);
}
+ public void incGroupCkNums(final String group, final String topic, final int incValue) {
+ final String statsKey = buildStatsKey(topic, group);
+ this.statsTable.get(GROUP_CK_NUMS).addValue(statsKey, incValue, 1);
+ }
+
+ public void incGroupAckNums(final String group, final String topic, final int incValue) {
+ final String statsKey = buildStatsKey(topic, group);
+ this.statsTable.get(GROUP_ACK_NUMS).addValue(statsKey, incValue, 1);
+ }
+
public String buildStatsKey(String topic, String group) {
StringBuilder strBuilder;
if (topic != null && group != null) {
@@@ -269,11 -487,16 +492,16 @@@
} else {
statsKey = buildStatsKey(topic, group);
}
- this.statsTable.get(GROUP_GET_LATENCY).addRTValue(statsKey, incValue, 1);
+ this.statsTable.get(Stats.GROUP_GET_LATENCY).addRTValue(statsKey, incValue, 1);
}
+ public void incTopicPutLatency(final String topic, final int queueId, final int incValue) {
+ final String statsKey = String.format("%d@%s", queueId, topic);
+ this.statsTable.get(TOPIC_PUT_LATENCY).addValue(statsKey, incValue, 1);
+ }
+
public void incBrokerPutNums() {
- this.statsTable.get(BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(1);
+ this.statsTable.get(Stats.BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(1);
}
public void incBrokerPutNums(final int incValue) {
@@@ -281,12 -504,20 +509,20 @@@
}
public void incBrokerGetNums(final int incValue) {
- this.statsTable.get(BROKER_GET_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
+ this.statsTable.get(Stats.BROKER_GET_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
}
+ public void incBrokerAckNums(final int incValue) {
+ this.statsTable.get(BROKER_ACK_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
+ }
+
+ public void incBrokerCkNums(final int incValue) {
+ this.statsTable.get(BROKER_CK_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
+ }
+
public void incSendBackNums(final String group, final String topic) {
final String statsKey = buildStatsKey(topic, group);
- this.statsTable.get(SNDBCK_PUT_NUMS).addValue(statsKey, 1, 1);
+ this.statsTable.get(Stats.SNDBCK_PUT_NUMS).addValue(statsKey, 1, 1);
}
public double tpsGroupGetNums(final String group, final String topic) {
diff --cc store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java
index 1b57b2d20,d299a47b3..9dc8a8b24
--- a/store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java
@@@ -56,9 -56,9 +56,9 @@@ public class AppendCallbackTest
messageStoreConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "unitteststore");
messageStoreConfig.setStorePathCommitLog(System.getProperty("user.home") + File.separator + "unitteststore" + File.separator + "commitlog");
//too much reference
- DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, null, null, null);
+ DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, null, null, new BrokerConfig());
CommitLog commitLog = new CommitLog(messageStore);
- callback = commitLog.new DefaultAppendMessageCallback(1024);
+ callback = commitLog.new DefaultAppendMessageCallback();
}
@After
diff --cc store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java
index 690ad0cbe,a9cd6c68c..6d42d6236
--- a/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java
@@@ -187,51 -185,26 +186,33 @@@ public class BatchPutMessageTest
}
}
+ private String generateKey(StringBuilder keyBuilder, MessageExt messageExt) {
+ keyBuilder.setLength(0);
+ keyBuilder.append(messageExt.getTopic());
+ keyBuilder.append('-');
+ keyBuilder.append(messageExt.getQueueId());
+ return keyBuilder.toString();
+ }
- @Test
- public void testPutLongBatchMessage() throws Exception{
- String topic = "batch-long-topic";
- MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) messageStore).getMessageStoreConfig();
- CommitLog commitLog = ((DefaultMessageStore) messageStore).getCommitLog();
- CommitLog.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get();
-
- MessageExtBatch messageExtBatch = new MessageExtBatch();
- messageExtBatch.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]);
- messageExtBatch.setTopic(topic);
- CommitLog.PutMessageContext putMessageContext = new CommitLog.PutMessageContext(generateKey(
- putMessageThreadLocal.getKeyBuilder(), messageExtBatch));
- RuntimeException runtimeException = assertThrows(RuntimeException.class,
- () -> putMessageThreadLocal.getEncoder().encode(messageExtBatch, putMessageContext));
- assertThat("message body size exceeded").isEqualTo(runtimeException.getMessage());
- }
-
-
private int calMsgLength(int bodyLength, int topicLength, int propertiesLength) {
final int msgLen = 4 //TOTALSIZE
- + 4 //MAGICCODE
- + 4 //BODYCRC
- + 4 //QUEUEID
- + 4 //FLAG
- + 8 //QUEUEOFFSET
- + 8 //PHYSICALOFFSET
- + 4 //SYSFLAG
- + 8 //BORNTIMESTAMP
- + 8 //BORNHOST
- + 8 //STORETIMESTAMP
- + 8 //STOREHOSTADDRESS
- + 4 //RECONSUMETIMES
- + 8 //Prepared Transaction Offset
- + 4 + (bodyLength > 0 ? bodyLength : 0) //BODY
- + 1 + topicLength //TOPIC
- + 2 + (propertiesLength > 0 ? propertiesLength : 0) //propertiesLength
- + 0;
+ + 4 //MAGICCODE
+ + 4 //BODYCRC
+ + 4 //QUEUEID
+ + 4 //FLAG
+ + 8 //QUEUEOFFSET
+ + 8 //PHYSICALOFFSET
+ + 4 //SYSFLAG
+ + 8 //BORNTIMESTAMP
+ + 8 //BORNHOST
+ + 8 //STORETIMESTAMP
+ + 8 //STOREHOSTADDRESS
+ + 4 //RECONSUMETIMES
+ + 8 //Prepared Transaction Offset
+ + 4 + (bodyLength > 0 ? bodyLength : 0) //BODY
+ + 1 + topicLength //TOPIC
+ + 2 + (propertiesLength > 0 ? propertiesLength : 0) //propertiesLength
+ + 0;
return msgLen;
}
diff --cc store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
index 491437d80,b39b9bb03..db9b3c665
--- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
@@@ -40,6 -46,9 +46,8 @@@ import org.apache.rocketmq.store.config
import org.apache.rocketmq.store.config.FlushDiskType;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
-import org.apache.rocketmq.store.logfile.DefaultMappedFile;
+ import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+ import org.apache.rocketmq.store.queue.CqUnit;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
import org.junit.After;
import org.junit.Before;
@@@ -644,80 -709,91 +708,151 @@@ public class DefaultMessageStoreTest
}
@Test
- public void testCleanUnusedLmqTopic() throws Exception {
- String lmqTopic = "%LMQ%123";
+ public void testPutMsgExceedsMaxLength() {
+ messageBody = new byte[4 * 1024 * 1024 + 1];
+ MessageExtBrokerInner msg = buildMessage();
- MessageExtBrokerInner messageExtBrokerInner = buildMessage();
- messageExtBrokerInner.setTopic("test");
- messageExtBrokerInner.setQueueId(0);
- messageExtBrokerInner.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqTopic);
- messageStore.putMessage(messageExtBrokerInner);
+ PutMessageResult result = messageStore.putMessage(msg);
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL);
+ }
+
+ @Test
+ public void testPutMsgBatchExceedsMaxLength() {
+ messageBody = new byte[4 * 1024 * 1024 + 1];
+ MessageExtBrokerInner msg1 = buildMessage();
+ MessageExtBrokerInner msg2 = buildMessage();
+ MessageExtBrokerInner msg3 = buildMessage();
+
+ MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3));
+ msgBatch.setBody(msgBatch.encode());
+
+ MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch);
+
+ try {
+ PutMessageResult result = this.messageStore.putMessages(msgExtBatch);
+ } catch (Exception e) {
- assertThat(e.getMessage()).contains("message size exceeded");
++ assertThat(e.getMessage()).contains("message body size exceeded");
+ }
+ }
+
+ @Test
+ public void testPutMsgWhenReplicasNotEnough() {
+ MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) this.messageStore).getMessageStoreConfig();
+ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
+ messageStoreConfig.setTotalReplicas(2);
+ messageStoreConfig.setInSyncReplicas(2);
+ messageStoreConfig.setEnableAutoInSyncReplicas(false);
+ ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);
+ this.messageStore.setAliveReplicaNumInGroup(1);
- Thread.sleep(3000);
- messageStore.cleanUnusedLmqTopic(lmqTopic);
+ MessageExtBrokerInner msg = buildMessage();
+ PutMessageResult result = this.messageStore.putMessage(msg);
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH);
+ ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);
+ }
+
+
+ @Test
+ public void testPutMsgWhenAdaptiveDegradation () {
+ MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) this.messageStore).getMessageStoreConfig();
+ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
+ messageStoreConfig.setTotalReplicas(2);
+ messageStoreConfig.setInSyncReplicas(2);
+ messageStoreConfig.setEnableAutoInSyncReplicas(true);
+ ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);
+ this.messageStore.setAliveReplicaNumInGroup(1);
+
+ MessageExtBrokerInner msg = buildMessage();
+ PutMessageResult result = this.messageStore.putMessage(msg);
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);
+ ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);
+ messageStoreConfig.setEnableAutoInSyncReplicas(false);
+ }
+
+
+ @Test
+ public void testGetBulkCommitLogData() {
+ DefaultMessageStore defaultMessageStore = (DefaultMessageStore) messageStore;
+
+ messageBody = new byte[2 * 1024 * 1024];
+
+ for (int i = 0; i < 10; i++) {
+ MessageExtBrokerInner msg1 = buildMessage();
+ messageStore.putMessage(msg1);
+ }
+
+ System.out.printf("%d%n", defaultMessageStore.getMaxPhyOffset());
+
+ List<SelectMappedBufferResult> bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset());
+ List<MessageExt> msgList = new ArrayList<>();
+ for (SelectMappedBufferResult bufferResult : bufferResultList) {
+ msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false));
+ bufferResult.release();
+ }
+ assertThat(msgList.size()).isEqualTo(10);
}
+ @Test
+ public void testPutLongMessage() throws Exception{
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
+ CommitLog commitLog = ((DefaultMessageStore) messageStore).getCommitLog();
+ MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) messageStore).getMessageStoreConfig();
+ CommitLog.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get();
+
+ //body size, topic size, properties size exactly equal to max size
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
+ messageExtBrokerInner.setTopic(new String(new byte[127]));
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));
+ PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult1 == null);
+
+ //body size exactly more than max message body size
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]);
+ PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult2.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+
+ //body size exactly equal to max message size
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]);
+ PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult3.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+
+ //message properties length more than properties maxSize
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE+1]));
+ PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult4.getPutMessageStatus() == PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);
+
+ //message length more than buffer length capacity
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
+ messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE]));
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));
+ PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult5.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+ }
+
+ @Test
+ public void testDynamicMaxMessageSize(){
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
+ MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) messageStore).getMessageStoreConfig();
+ int originMaxMessageSize = messageStoreConfig.getMaxMessageSize();
+
+ messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]);
+ PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+
+ int newMaxMessageSize = originMaxMessageSize + 10;
+ messageStoreConfig.setMaxMessageSize(newMaxMessageSize);
+ putMessageResult = messageStore.putMessage(messageExtBrokerInner);
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK);
+
+ messageStoreConfig.setMaxMessageSize(10);
+ putMessageResult = messageStore.putMessage(messageExtBrokerInner);
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+
+ messageStoreConfig.setMaxMessageSize(originMaxMessageSize);
+ }
+
private class MyMessageArrivingListener implements MessageArrivingListener {
@Override
public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,
diff --cc store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
index cdf3f1610,627f9dd2a..a7e726c57
--- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
@@@ -27,8 -31,9 +31,8 @@@ import org.junit.After
import org.junit.Before;
import org.junit.Test;
+ import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue;
import static org.junit.Assert.assertEquals;
--import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
diff --cc store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java
index 5660de136,7a2e32a21..9dc724cd3
--- a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java
+++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java
@@@ -20,6 -20,8 +20,7 @@@ import org.apache.rocketmq.common.UtilA
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExtBatch;
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-import org.apache.rocketmq.store.logfile.DefaultMappedFile;
import org.junit.After;
import java.io.File;
diff --cc test/pom.xml
index 0a6f5edb4,3cf4f2d61..c3b2db596
--- a/test/pom.xml
+++ b/test/pom.xml
@@@ -40,10 -40,10 +40,14 @@@
<groupId>${project.groupId}</groupId>
<artifactId>rocketmq-namesrv</artifactId>
</dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-container</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>annotations-api</artifactId>
+ </dependency>
<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
diff --cc test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
index 6a13049f7,6a13049f7..a186e73fe
--- a/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
+++ b/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
@@@ -29,7 -29,7 +29,7 @@@ import org.apache.rocketmq.test.util.Te
public class AbstractListener extends MQCollector implements MessageListener {
public static Logger logger = Logger.getLogger(AbstractListener.class);
-- protected boolean isDebug = false;
++ protected boolean isDebug = true;
protected String listenerName = null;
protected Collection<Object> allSendMsgs = null;
diff --cc test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
index 000000000,1d8098053..d50ff718f
mode 000000,100644..100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
@@@ -1,0 -1,97 +1,96 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.rocketmq.test.client.consumer.pop;
+
+ import org.apache.rocketmq.common.message.MessageClientExt;
+ import org.apache.rocketmq.common.message.MessageConst;
+ import org.apache.rocketmq.common.message.MessageRequestMode;
+ import org.apache.rocketmq.logging.inner.Logger;
+ import org.apache.rocketmq.test.base.BaseConf;
+ import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
+ import org.apache.rocketmq.test.client.rmq.RMQPopConsumer;
+ import org.apache.rocketmq.test.factory.ConsumerFactory;
+ import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
+ import org.apache.rocketmq.test.util.RandomUtil;
+ import org.apache.rocketmq.test.util.TestUtils;
+ import org.apache.rocketmq.test.util.VerifyUtils;
+ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+ import org.junit.After;
+ import org.junit.Assert;
+ import org.junit.Before;
-import org.junit.Ignore;
+ import org.junit.Test;
+
+ import static com.google.common.truth.Truth.assertThat;
+
+ public class PopSubCheckIT extends BaseConf {
+ private static Logger logger = Logger.getLogger(PopSubCheckIT.class);
+ private String group;
+
+ private DefaultMQAdminExt defaultMQAdminExt;
+
+ @Before
+ public void setUp() throws Exception {
+ group = initConsumerGroup();
+
+ defaultMQAdminExt = new DefaultMQAdminExt();
+ defaultMQAdminExt.setInstanceName(RandomUtil.getStringByUUID());
+ defaultMQAdminExt.start();
+ }
+
+ @After
+ public void tearDown() {
+ defaultMQAdminExt.shutdown();
+ super.shutdown();
+ }
+
- @Ignore
++
+ @Test
+ public void testNormalPopAck() throws Exception {
+ String topic = initTopic();
+ logger.info(String.format("use topic: %s; group: %s !", topic, group));
+
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ producer.getProducer().setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);
+
+ for (String brokerAddr : new String[]{brokerController1.getBrokerAddr(), brokerController2.getBrokerAddr()}) {
+ defaultMQAdminExt.setMessageRequestMode(brokerAddr, topic, group, MessageRequestMode.POP, 8, 60_000);
+ }
+
+ RMQPopConsumer consumer = ConsumerFactory.getRMQPopConsumer(nsAddr, group,
+ topic, "*", new RMQNormalListener());
+ mqClients.add(consumer);
+
+ int msgNum = 1;
+ producer.send(msgNum);
+ Assert.assertEquals("Not all sent succeeded", msgNum, producer.getAllUndupMsgBody().size());
+ logger.info(producer.getFirstMsg());
+
+ TestUtils.waitForSeconds(10);
+
+ consumer.getListener().waitForMessageConsume(msgNum, 30_000);
+ assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(), consumer.getListener().getAllMsgBody()))
+ .containsExactlyElementsIn(producer.getAllMsgBody());
+ for (Object o : consumer.getListener().getAllOriginMsg()) {
+ MessageClientExt msg = (MessageClientExt) o;
+ assertThat(msg.getProperty(MessageConst.PROPERTY_POP_CK)).named("check pop meta").isNotEmpty();
+ }
+
+ consumer.getListener().waitForMessageConsume(msgNum, 3_000 * 9);
+ assertThat(consumer.getListener().getAllOriginMsg().size()).isEqualTo(msgNum);
+ }
+ }
diff --cc test/src/test/java/org/apache/rocketmq/test/container/ContainerIntegrationTestBase.java
index 000000000,4a2e06642..e1cadaf45
mode 000000,100644..100644
--- a/test/src/test/java/org/apache/rocketmq/test/container/ContainerIntegrationTestBase.java
+++ b/test/src/test/java/org/apache/rocketmq/test/container/ContainerIntegrationTestBase.java
@@@ -1,0 -1,666 +1,666 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.rocketmq.test.container;
+
+ import ch.qos.logback.classic.LoggerContext;
+ import ch.qos.logback.classic.joran.JoranConfigurator;
+ import io.netty.channel.ChannelHandlerContext;
+ import java.io.File;
+ import java.io.IOException;
+ import java.nio.file.Files;
+ import java.time.Duration;
+ import java.util.ArrayList;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Random;
+ import java.util.Set;
+ import java.util.UUID;
+ import java.util.concurrent.ConcurrentHashMap;
+ import java.util.concurrent.ConcurrentMap;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicInteger;
+ import org.apache.commons.lang3.builder.EqualsBuilder;
+ import org.apache.commons.lang3.builder.HashCodeBuilder;
+ import org.apache.rocketmq.container.BrokerContainer;
+ import org.apache.rocketmq.container.InnerSalveBrokerController;
+ import org.apache.rocketmq.broker.BrokerController;
+ import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+ import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+ import org.apache.rocketmq.client.producer.DefaultMQProducer;
+ import org.apache.rocketmq.client.producer.TransactionCheckListener;
+ import org.apache.rocketmq.client.producer.TransactionMQProducer;
+ import org.apache.rocketmq.common.BrokerConfig;
+ import org.apache.rocketmq.container.BrokerContainerConfig;
+ import org.apache.rocketmq.common.BrokerIdentity;
+ import org.apache.rocketmq.common.MQVersion;
+ import org.apache.rocketmq.common.TopicConfig;
+ import org.apache.rocketmq.common.UtilAll;
+ import org.apache.rocketmq.common.message.MessageQueue;
+ import org.apache.rocketmq.common.namesrv.NamesrvConfig;
+ import org.apache.rocketmq.common.protocol.RequestCode;
+ import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader;
+ import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+ import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.logging.InternalLoggerFactory;
+ import org.apache.rocketmq.namesrv.NamesrvController;
+ import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+ import org.apache.rocketmq.store.config.BrokerRole;
+ import org.apache.rocketmq.store.config.MessageStoreConfig;
+ import org.apache.rocketmq.store.ha.HAConnection;
+ import org.apache.rocketmq.store.ha.HAConnectionState;
+ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+ import org.junit.Assert;
+ import org.junit.BeforeClass;
+ import org.slf4j.LoggerFactory;
+
+ import static org.awaitility.Awaitility.await;
+
+ /**
+ * ContainerIntegrationTestBase will setup a rocketmq ha cluster contains two broker group:
+ * <li>BrokerA contains two replicas</li>
+ * <li>BrokerB contains three replicas</li>
+ */
+ public class ContainerIntegrationTestBase {
+ private static final AtomicBoolean CLUSTER_SET_UP = new AtomicBoolean(false);
+ private static final List<File> TMP_FILE_LIST = new ArrayList<>();
+ private static final Random RANDOM = new Random();
+ protected static String nsAddr;
+
+ protected static final String THREE_REPLICAS_TOPIC = "SEND_MESSAGE_TEST_TOPIC_THREE_REPLICAS";
+
+ protected static final List<BrokerContainer> brokerContainerList = new ArrayList<>();
+ protected static final List<NamesrvController> namesrvControllers = new ArrayList<>();
+
+ protected static final String BROKER_NAME_PREFIX = "TestBrokerName_";
+ protected static final int COMMIT_LOG_SIZE = 128 * 1024;
+ protected static final int INDEX_NUM = 1000;
+ protected static final AtomicInteger BROKER_INDEX = new AtomicInteger(0);
+
+ protected static BrokerContainer brokerContainer1;
+ protected static BrokerContainer brokerContainer2;
+ protected static BrokerContainer brokerContainer3;
+ protected static BrokerController master1With3Replicas;
+ protected static BrokerController master2With3Replicas;
+ protected static BrokerController master3With3Replicas;
+ protected static NamesrvController namesrvController;
+
+ protected static DefaultMQAdminExt defaultMQAdminExt;
+
+ private final static InternalLogger LOG = InternalLoggerFactory.getLogger(ContainerIntegrationTestBase.class);
+ private static ConcurrentMap<BrokerConfig, MessageStoreConfig> slaveStoreConfigCache = new ConcurrentHashMap<>();
+
+ protected static ConcurrentMap<BrokerConfigLite, BrokerController> isolatedBrokers = new ConcurrentHashMap<>();
+ private static final Set<Integer> PORTS_IN_USE = new HashSet<>();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (CLUSTER_SET_UP.compareAndSet(false, true)) {
+ System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));
+ System.setProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "0.99");
+ System.setProperty("rocketmq.broker.diskSpaceWarningLevelRatio", "0.99");
+
+ LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
+ JoranConfigurator configurator = new JoranConfigurator();
+ configurator.setContext(lc);
+ lc.reset();
+ //https://logback.qos.ch/manual/configuration.html
+ lc.setPackagingDataEnabled(false);
+
+ configurator.doConfigure("../distribution/conf/logback_broker.xml");
+ configurator.doConfigure("../distribution/conf/logback_namesrv.xml");
+
+ setUpCluster();
+ setUpTopic();
+ registerCleaner();
+
+ System.out.printf("cluster setup complete%n");
+ }
+ }
+
+ private static void setUpTopic() {
+ createTopic(THREE_REPLICAS_TOPIC);
+ }
+
+ private static void createTopic(String topic) {
+ createTopicTo(master1With3Replicas, topic);
+ createTopicTo(master2With3Replicas, topic);
+ createTopicTo(master3With3Replicas, topic);
+ }
+
+ private static void setUpCluster() throws Exception {
+ namesrvController = createAndStartNamesrv();
+ nsAddr = "127.0.0.1:" + namesrvController.getNettyServerConfig().getListenPort();
+ System.out.printf("namesrv addr: %s%n", nsAddr);
+
+ /*
+ * BrokerContainer1 | BrokerContainer2 | BrokerContainer3
+ *
+ * master1With3Replicas(m) master2With3Replicas(m) master3With3Replicas(m)
+ * master3With3Replicas(s0) master1With3Replicas(s0) master2With3Replicas(s0)
+ * master2With3Replicas(s1) master3With3Replicas(s1) master1With3Replicas(s1)
+ */
+
+ brokerContainer1 = createAndStartBrokerContainer(nsAddr);
+ brokerContainer2 = createAndStartBrokerContainer(nsAddr);
+ brokerContainer3 = createAndStartBrokerContainer(nsAddr);
+ // Create three broker groups, two contains two replicas, another contains three replicas
+ master1With3Replicas = createAndAddMaster(brokerContainer1, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());
+ master2With3Replicas = createAndAddMaster(brokerContainer2, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());
+ master3With3Replicas = createAndAddMaster(brokerContainer3, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());
+
+ createAndAddSlave(1, brokerContainer1, master3With3Replicas);
+ createAndAddSlave(1, brokerContainer2, master1With3Replicas);
+ createAndAddSlave(1, brokerContainer3, master2With3Replicas);
+
+ createAndAddSlave(2, brokerContainer1, master2With3Replicas);
+ createAndAddSlave(2, brokerContainer2, master3With3Replicas);
+ createAndAddSlave(2, brokerContainer3, master1With3Replicas);
+
+ awaitUntilSlaveOK();
+
+ defaultMQAdminExt = new DefaultMQAdminExt("HATest_Admin_Group");
+ defaultMQAdminExt.setNamesrvAddr(nsAddr);
+ defaultMQAdminExt.start();
+ }
+
+ protected static void createTopicTo(BrokerController masterBroker, String topicName, int rqn, int wqn) {
+ try {
+ TopicConfig topicConfig = new TopicConfig(topicName, rqn, wqn, 6, 0);
+ defaultMQAdminExt.createAndUpdateTopicConfig(masterBroker.getBrokerAddr(), topicConfig);
+
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer1);
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer2);
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer3);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException("Create topic to broker failed", e);
+ }
+ }
+
+ protected static void createGroup(BrokerController masterBroker, String groupName) {
+ try {
+ SubscriptionGroupConfig config = new SubscriptionGroupConfig();
+ config.setGroupName(groupName);
+
+ masterBroker.getSubscriptionGroupManager().updateSubscriptionGroupConfig(config);
+
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer1);
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer2);
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer3);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException("Create group to broker failed", e);
+ }
+ }
+
+ private static void triggerSlaveSync(String brokerName, BrokerContainer brokerContainer) {
+ for (InnerSalveBrokerController slaveBroker : brokerContainer.getSlaveBrokers()) {
+ if (slaveBroker.getBrokerConfig().getBrokerName().equals(brokerName)) {
+ slaveBroker.getSlaveSynchronize().syncAll();
+ slaveBroker.registerBrokerAll(true, false, true);
+ }
+ }
+ }
+
+ protected static void createTopicTo(BrokerController brokerController, String topicName) {
+ createTopicTo(brokerController, topicName, 8, 8);
+ }
+
+ private static void registerCleaner() {
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ if (CLUSTER_SET_UP.compareAndSet(true, false)) {
+ System.out.printf("clean up%n");
+ defaultMQAdminExt.shutdown();
+
+ for (final BrokerContainer brokerContainer : brokerContainerList) {
+ brokerContainer.shutdown();
+ for (BrokerController brokerController : brokerContainer.getBrokerControllers()) {
+ brokerController.getMessageStore().destroy();
+ }
+ }
+
+ for (final NamesrvController namesrvController : namesrvControllers) {
+ namesrvController.shutdown();
+ }
+
+ for (final File file : TMP_FILE_LIST) {
+ UtilAll.deleteFile(file);
+ }
+ }
+ }));
+ }
+
+ private static File createBaseDir(String prefix) {
+ final File file;
+ try {
+ file = Files.createTempDirectory(prefix).toFile();
+ TMP_FILE_LIST.add(file);
+ System.out.printf("create file at %s%n", file.getAbsolutePath());
+ return file;
+ } catch (IOException e) {
+ throw new RuntimeException("Couldn't create tmp folder", e);
+ }
+ }
+
+ public static NamesrvController createAndStartNamesrv() {
+ String baseDir = createBaseDir("test-cluster-namesrv").getAbsolutePath();
+ NamesrvConfig namesrvConfig = new NamesrvConfig();
+ NettyServerConfig nameServerNettyServerConfig = new NettyServerConfig();
+ namesrvConfig.setKvConfigPath(baseDir + File.separator + "namesrv" + File.separator + "kvConfig.json");
+ namesrvConfig.setConfigStorePath(baseDir + File.separator + "namesrv" + File.separator + "namesrv.properties");
+ namesrvConfig.setSupportActingMaster(true);
+ namesrvConfig.setScanNotActiveBrokerInterval(1000);
+
+ nameServerNettyServerConfig.setListenPort(generatePort(10000, 10000));
+ NamesrvController namesrvController = new NamesrvController(namesrvConfig, nameServerNettyServerConfig);
+ try {
+ Assert.assertTrue(namesrvController.initialize());
+ LOG.info("Name Server Start:{}", nameServerNettyServerConfig.getListenPort());
+ namesrvController.start();
+ } catch (Exception e) {
+ LOG.info("Name Server start failed");
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ namesrvController.getRemotingServer().registerProcessor(RequestCode.REGISTER_BROKER, new NettyRequestProcessor() {
+ @Override
+ public RemotingCommand processRequest(final ChannelHandlerContext ctx,
+ final RemotingCommand request) throws Exception {
- final RegisterBrokerRequestHeader requestHeader = request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class);
++ final RegisterBrokerRequestHeader requestHeader = (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class);
+ final BrokerConfigLite liteConfig = new BrokerConfigLite(requestHeader.getClusterName(),
+ requestHeader.getBrokerName(),
+ requestHeader.getBrokerAddr(),
+ requestHeader.getBrokerId());
+ if (isolatedBrokers.containsKey(liteConfig)) {
+ // return response with SYSTEM_ERROR
+ return RemotingCommand.createResponseCommand(null);
+ }
+ return namesrvController.getRemotingServer().getDefaultProcessorPair().getObject1().processRequest(ctx, request);
+ }
+
+ @Override
+ public boolean rejectRequest() {
+ return false;
+ }
+ }, null);
+
+ namesrvControllers.add(namesrvController);
+ return namesrvController;
+
+ }
+
+ public static BrokerContainer createAndStartBrokerContainer(String nsAddr) {
+ BrokerContainerConfig brokerContainerConfig = new BrokerContainerConfig();
+ NettyServerConfig nettyServerConfig = new NettyServerConfig();
+ NettyClientConfig nettyClientConfig = new NettyClientConfig();
+ brokerContainerConfig.setNamesrvAddr(nsAddr);
+
+ nettyServerConfig.setListenPort(generatePort(20000, 10000));
+ BrokerContainer brokerContainer = new BrokerContainer(brokerContainerConfig, nettyServerConfig, nettyClientConfig);
+ try {
+ Assert.assertTrue(brokerContainer.initialize());
+ LOG.info("Broker container Start, listen on {}.", nettyServerConfig.getListenPort());
+ brokerContainer.start();
+ } catch (Exception e) {
+ LOG.info("Broker container start failed", e);
+ e.printStackTrace();
+ System.exit(1);
+ }
+ brokerContainerList.add(brokerContainer);
+ return brokerContainer;
+ }
+
+ private static int generatePort(int base, int range) {
+ int result = base + RANDOM.nextInt(range);
+ while (PORTS_IN_USE.contains(result) || PORTS_IN_USE.contains(result - 2)) {
+ result = base + RANDOM.nextInt(range);
+ }
+ PORTS_IN_USE.add(result);
+ PORTS_IN_USE.add(result - 2);
+ return result;
+ }
+
+ public static BrokerController createAndAddMaster(BrokerContainer brokerContainer,
+ BrokerGroupConfig brokerGroupConfig, int brokerIndex) throws Exception {
+ BrokerConfig brokerConfig = new BrokerConfig();
+ MessageStoreConfig storeConfig = new MessageStoreConfig();
+ brokerConfig.setBrokerName(BROKER_NAME_PREFIX + brokerIndex);
+ brokerConfig.setBrokerIP1("127.0.0.1");
+ brokerConfig.setBrokerIP2("127.0.0.1");
+ brokerConfig.setBrokerId(0);
+ brokerConfig.setEnablePropertyFilter(true);
+ brokerConfig.setEnableSlaveActingMaster(brokerGroupConfig.enableSlaveActingMaster);
+ brokerConfig.setEnableRemoteEscape(brokerGroupConfig.enableRemoteEscape);
+ brokerConfig.setSlaveReadEnable(brokerGroupConfig.slaveReadEnable);
+ brokerConfig.setLockInStrictMode(true);
+ brokerConfig.setConsumerOffsetUpdateVersionStep(10);
+ brokerConfig.setDelayOffsetUpdateVersionStep(10);
+ brokerConfig.setCompatibleWithOldNameSrv(false);
+ brokerConfig.setListenPort(generatePort(brokerContainer.getRemotingServer().localListenPort(), 10000));
+
+ String baseDir = createBaseDir(brokerConfig.getBrokerName() + "_" + brokerConfig.getBrokerId()).getAbsolutePath();
+ storeConfig.setStorePathRootDir(baseDir);
+ storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog");
+ storeConfig.setHaListenPort(generatePort(30000, 10000));
+ storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE);
+ storeConfig.setMaxIndexNum(INDEX_NUM);
+ storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);
+ storeConfig.setTotalReplicas(brokerGroupConfig.totalReplicas);
+ storeConfig.setInSyncReplicas(brokerGroupConfig.inSyncReplicas);
+ storeConfig.setMinInSyncReplicas(brokerGroupConfig.minReplicas);
+ storeConfig.setEnableAutoInSyncReplicas(brokerGroupConfig.autoReplicas);
+ storeConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
+ storeConfig.setSyncFlushTimeout(10 * 1000);
+
+ System.out.printf("start master %s with port %d-%d%n", brokerConfig.getCanonicalName(), brokerConfig.getListenPort(), storeConfig.getHaListenPort());
+ BrokerController brokerController = null;
+ try {
+ brokerController = brokerContainer.addBroker(brokerConfig, storeConfig);
+ Assert.assertNotNull(brokerController);
+ brokerController.start();
+ TMP_FILE_LIST.add(new File(brokerController.getTopicConfigManager().configFilePath()));
+ TMP_FILE_LIST.add(new File(brokerController.getSubscriptionGroupManager().configFilePath()));
+ LOG.info("Broker Start name:{} addr:{}", brokerConfig.getBrokerName(), brokerController.getBrokerAddr());
+ } catch (Exception e) {
+ LOG.info("Broker start failed", e);
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ return brokerController;
+ }
+
+ protected static DefaultMQProducer createProducer(String producerGroup) {
+ DefaultMQProducer producer = new DefaultMQProducer(producerGroup);
+ producer.setInstanceName(UUID.randomUUID().toString());
+ producer.setNamesrvAddr(nsAddr);
+ return producer;
+ }
+
+ protected static TransactionMQProducer createTransactionProducer(String producerGroup,
+ TransactionCheckListener transactionCheckListener) {
+ TransactionMQProducer producer = new TransactionMQProducer(producerGroup);
+ producer.setInstanceName(UUID.randomUUID().toString());
+ producer.setNamesrvAddr(nsAddr);
+ producer.setTransactionCheckListener(transactionCheckListener);
+ return producer;
+ }
+
+ protected static DefaultMQPullConsumer createPullConsumer(String consumerGroup) {
+ DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);
+ consumer.setInstanceName(UUID.randomUUID().toString());
+ consumer.setNamesrvAddr(nsAddr);
+ return consumer;
+ }
+
+ protected static DefaultMQPushConsumer createPushConsumer(String consumerGroup) {
+ DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup);
+ consumer.setInstanceName(UUID.randomUUID().toString());
+ consumer.setNamesrvAddr(nsAddr);
+ return consumer;
+ }
+
+ protected static void createAndAddSlave(int slaveBrokerId, BrokerContainer brokerContainer,
+ BrokerController master) {
+ BrokerConfig slaveBrokerConfig = new BrokerConfig();
+ slaveBrokerConfig.setBrokerName(master.getBrokerConfig().getBrokerName());
+ slaveBrokerConfig.setBrokerId(slaveBrokerId);
+ slaveBrokerConfig.setBrokerClusterName(master.getBrokerConfig().getBrokerClusterName());
+ slaveBrokerConfig.setCompatibleWithOldNameSrv(false);
+ slaveBrokerConfig.setBrokerIP1("127.0.0.1");
+ slaveBrokerConfig.setBrokerIP2("127.0.0.1");
+ slaveBrokerConfig.setEnablePropertyFilter(true);
+ slaveBrokerConfig.setSlaveReadEnable(true);
+ slaveBrokerConfig.setEnableSlaveActingMaster(true);
+ slaveBrokerConfig.setEnableRemoteEscape(true);
+ slaveBrokerConfig.setLockInStrictMode(true);
+ slaveBrokerConfig.setListenPort(generatePort(brokerContainer.getRemotingServer().localListenPort(), 10000));
+ slaveBrokerConfig.setConsumerOffsetUpdateVersionStep(10);
+ slaveBrokerConfig.setDelayOffsetUpdateVersionStep(10);
+
+ MessageStoreConfig storeConfig = slaveStoreConfigCache.get(slaveBrokerConfig);
+
+ if (storeConfig == null) {
+ storeConfig = new MessageStoreConfig();
+ String baseDir = createBaseDir(slaveBrokerConfig.getBrokerName() + "_" + slaveBrokerConfig.getBrokerId()).getAbsolutePath();
+ storeConfig.setStorePathRootDir(baseDir);
+ storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog");
+ storeConfig.setHaListenPort(generatePort(master.getMessageStoreConfig().getHaListenPort(), 10000));
+ storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE);
+ storeConfig.setMaxIndexNum(INDEX_NUM);
+ storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);
+ storeConfig.setTotalReplicas(master.getMessageStoreConfig().getTotalReplicas());
+ storeConfig.setInSyncReplicas(master.getMessageStoreConfig().getInSyncReplicas());
+ storeConfig.setMinInSyncReplicas(master.getMessageStoreConfig().getMinInSyncReplicas());
+ storeConfig.setBrokerRole(BrokerRole.SLAVE);
+ slaveStoreConfigCache.put(slaveBrokerConfig, storeConfig);
+ }
+
+ System.out.printf("start slave %s with port %d-%d%n", slaveBrokerConfig.getCanonicalName(), slaveBrokerConfig.getListenPort(), storeConfig.getHaListenPort());
+
+ try {
+ BrokerController brokerController = brokerContainer.addBroker(slaveBrokerConfig, storeConfig);
+ Assert.assertNotNull(brokerContainer);
+ brokerController.start();
+ TMP_FILE_LIST.add(new File(brokerController.getTopicConfigManager().configFilePath()));
+ TMP_FILE_LIST.add(new File(brokerController.getSubscriptionGroupManager().configFilePath()));
+ LOG.info("Add slave name:{} addr:{}", slaveBrokerConfig.getBrokerName(), brokerController.getBrokerAddr());
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException("Couldn't add slave broker", e);
+ }
+ }
+
+ protected static void removeSlaveBroker(int slaveBrokerId, BrokerContainer brokerContainer,
+ BrokerController master) throws Exception {
+ BrokerIdentity brokerIdentity = new BrokerIdentity(master.getBrokerConfig().getBrokerClusterName(),
+ master.getBrokerConfig().getBrokerName(), slaveBrokerId);
+
+ brokerContainer.removeBroker(brokerIdentity);
+ }
+
+ protected static void awaitUntilSlaveOK() {
+ await().atMost(100, TimeUnit.SECONDS)
+ .until(() -> {
+ boolean isOk = master1With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2
+ && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;
+ for (HAConnection haConnection : master1With3Replicas.getMessageStore().getHaService().getConnectionList()) {
+ isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);
+ }
+ return isOk;
+ });
+
+ await().atMost(100, TimeUnit.SECONDS)
+ .until(() -> {
+ boolean isOk = master2With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2
+ && master2With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;
+ for (HAConnection haConnection : master2With3Replicas.getMessageStore().getHaService().getConnectionList()) {
+ isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);
+ }
+ return isOk;
+ });
+
+ await().atMost(100, TimeUnit.SECONDS)
+ .until(() -> {
+ boolean isOk = master3With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2
+ && master3With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;
+ for (HAConnection haConnection : master3With3Replicas.getMessageStore().getHaService().getConnectionList()) {
+ isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);
+ }
+ return isOk;
+ });
+
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ protected static void isolateBroker(BrokerController brokerController) {
+ final BrokerConfig config = brokerController.getBrokerConfig();
+
+ BrokerConfigLite liteConfig = new BrokerConfigLite(config.getBrokerClusterName(),
+ config.getBrokerName(),
+ brokerController.getBrokerAddr(),
+ config.getBrokerId());
+
+ // Reject register requests from the specific broker
+ isolatedBrokers.putIfAbsent(liteConfig, brokerController);
+
+ // UnRegister the specific broker immediately
+ namesrvController.getRouteInfoManager().unregisterBroker(liteConfig.getClusterName(),
+ liteConfig.getBrokerAddr(),
+ liteConfig.getBrokerName(),
+ liteConfig.getBrokerId());
+ }
+
+ protected static void cancelIsolatedBroker(BrokerController brokerController) {
+ final BrokerConfig config = brokerController.getBrokerConfig();
+
+ BrokerConfigLite liteConfig = new BrokerConfigLite(config.getBrokerClusterName(),
+ config.getBrokerName(),
+ brokerController.getBrokerAddr(),
+ config.getBrokerId());
+
+ isolatedBrokers.remove(liteConfig);
+ brokerController.registerBrokerAll(true, false, true);
+
+ await().atMost(Duration.ofMinutes(1)).until(() -> namesrvController.getRouteInfoManager()
+ .getBrokerMemberGroup(liteConfig.getClusterName(), liteConfig.brokerName).getBrokerAddrs()
+ .containsKey(liteConfig.getBrokerId()));
+ }
+
+ protected static InnerSalveBrokerController getSlaveFromContainerByName(BrokerContainer brokerContainer,
+ String brokerName) {
+ InnerSalveBrokerController targetSlave = null;
+ for (InnerSalveBrokerController slave : brokerContainer.getSlaveBrokers()) {
+ if (slave.getBrokerConfig().getBrokerName().equals(brokerName)) {
+ targetSlave = slave;
+ }
+ }
+
+ return targetSlave;
+ }
+
+ protected static void changeCompatibleMode(boolean compatibleMode) {
+ brokerContainer1.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));
+ brokerContainer2.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));
+ brokerContainer3.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));
+ }
+
+ protected static Set<MessageQueue> filterMessageQueue(Set<MessageQueue> mqSet, String topic) {
+ Set<MessageQueue> targetMqSet = new HashSet<>();
+ if (topic != null) {
+ for (MessageQueue mq : mqSet) {
+ if (mq.getTopic().equals(topic)) {
+ targetMqSet.add(mq);
+ }
+ }
+ }
+
+ return targetMqSet;
+ }
+
+ public static class BrokerGroupConfig {
+ int totalReplicas = 3;
+ int minReplicas = 1;
+ int inSyncReplicas = 2;
+ boolean autoReplicas = true;
+ boolean enableSlaveActingMaster = true;
+ boolean enableRemoteEscape = true;
+ boolean slaveReadEnable = true;
+
+ public BrokerGroupConfig() {
+ }
+
+ public BrokerGroupConfig(final int totalReplicas, final int minReplicas, final int inSyncReplicas,
+ final boolean autoReplicas, boolean enableSlaveActingMaster, boolean slaveReadEnable) {
+ this.totalReplicas = totalReplicas;
+ this.minReplicas = minReplicas;
+ this.inSyncReplicas = inSyncReplicas;
+ this.autoReplicas = autoReplicas;
+ this.enableSlaveActingMaster = enableSlaveActingMaster;
+ this.slaveReadEnable = slaveReadEnable;
+ }
+ }
+
+ static class BrokerConfigLite {
+ private String clusterName;
+ private String brokerName;
+ private String brokerAddr;
+ private long brokerId;
+
+ public BrokerConfigLite(final String clusterName, final String brokerName, final String brokerAddr,
+ final long brokerId) {
+ this.clusterName = clusterName;
+ this.brokerName = brokerName;
+ this.brokerAddr = brokerAddr;
+ this.brokerId = brokerId;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public String getBrokerName() {
+ return brokerName;
+ }
+
+ public String getBrokerAddr() {
+ return brokerAddr;
+ }
+
+ public long getBrokerId() {
+ return brokerId;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ final BrokerConfigLite lite = (BrokerConfigLite) o;
+
+ return new EqualsBuilder()
+ .append(clusterName, lite.clusterName)
+ .append(brokerName, lite.brokerName)
+ .append(brokerAddr, lite.brokerAddr)
+ .append(brokerId, lite.brokerId)
+ .isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 37)
+ .append(clusterName)
+ .append(brokerName)
+ .append(brokerAddr)
+ .append(brokerId)
+ .toHashCode();
+ }
+ }
+ }
diff --cc test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
index 1550cf949,1550cf949..ce2829d28
--- a/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
@@@ -19,7 -19,7 +19,6 @@@ package org.apache.rocketmq.test.delay
import java.util.List;
import org.apache.log4j.Logger;
--import org.apache.rocketmq.test.client.consumer.balance.NormalMsgStaticBalanceIT;
import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.factory.MQMessageFactory;
diff --cc test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java
index 000000000,41c9c7e4f..d2f7e81ba
mode 000000,100644..100644
--- a/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java
@@@ -1,0 -1,522 +1,521 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.rocketmq.test.statictopic;
+
+ import com.google.common.collect.ImmutableList;
+ import com.google.common.collect.ImmutableSet;
+ import org.apache.log4j.Logger;
+ import org.apache.rocketmq.broker.BrokerController;
+ import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+ import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+ import org.apache.rocketmq.client.producer.DefaultMQProducer;
+ import org.apache.rocketmq.common.MixAll;
+ import org.apache.rocketmq.common.admin.ConsumeStats;
+ import org.apache.rocketmq.common.admin.OffsetWrapper;
+ import org.apache.rocketmq.common.admin.TopicStatsTable;
+ import org.apache.rocketmq.common.message.MessageExt;
+ import org.apache.rocketmq.common.message.MessageQueue;
+ import org.apache.rocketmq.common.rpc.ClientMetadata;
+ import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
+ import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingOne;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
+ import org.apache.rocketmq.test.base.BaseConf;
+ import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
+ import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
+ import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
+ import org.apache.rocketmq.test.util.MQAdminTestUtils;
+ import org.apache.rocketmq.test.util.MQRandomUtils;
+ import org.apache.rocketmq.test.util.TestUtils;
+ import org.apache.rocketmq.test.util.VerifyUtils;
+ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+ import org.apache.rocketmq.tools.admin.MQAdminUtils;
+ import org.junit.After;
+ import org.junit.Assert;
+ import org.junit.Before;
+ import org.junit.FixMethodOrder;
-import org.junit.Ignore;
+ import org.junit.Test;
+
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.Comparator;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+
+ import static com.google.common.truth.Truth.assertThat;
+ import static org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils.getMappingDetailFromConfig;
+
+ @FixMethodOrder
+ public class StaticTopicIT extends BaseConf {
+
+ private static Logger logger = Logger.getLogger(StaticTopicIT.class);
+ private DefaultMQAdminExt defaultMQAdminExt;
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty("rocketmq.client.rebalance.waitInterval", "500");
+ defaultMQAdminExt = getAdmin(nsAddr);
+ waitBrokerRegistered(nsAddr, clusterName, brokerNum);
+ defaultMQAdminExt.start();
+ }
+
+
+ @Test
+ public void testCommandsWithCluster() throws Exception {
+ //This case is used to mock the env to test the command manually
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener());
+ int queueNum = 10;
+ int msgEachQueue = 100;
+
+ {
+ MQAdminTestUtils.createStaticTopicWithCommand(topic, queueNum, null, clusterName, nsAddr);
+ sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 0);
+ //consume and check
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+ {
+ MQAdminTestUtils.remappingStaticTopicWithCommand(topic, null, clusterName, nsAddr);
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);
+ sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, msgEachQueue);
+ }
+ }
+
+ @Test
+ public void testCommandsWithBrokers() throws Exception {
+ //This case is used to mock the env to test the command manually
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener());
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ {
+ Set<String> brokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopicWithCommand(topic, queueNum, brokers, null, nsAddr);
+ sendMessagesAndCheck(producer, brokers, topic, queueNum, msgEachQueue, 0);
+ //consume and check
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+ {
+ Set<String> brokers = ImmutableSet.of(broker2Name);
+ MQAdminTestUtils.remappingStaticTopicWithCommand(topic, brokers, null, nsAddr);
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);
+ sendMessagesAndCheck(producer, brokers, topic, queueNum, msgEachQueue, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 2);
+ }
+ }
+
+ @Test
+ public void testNoTargetBrokers() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ int queueNum = 10;
+ {
+ Set<String> targetBrokers = new HashSet<>();
+ targetBrokers.add(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size());
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ TopicConfigAndQueueMapping configMapping = remoteBrokerConfigMap.get(broker2Name);
+ Assert.assertEquals(0, configMapping.getWriteQueueNums());
+ Assert.assertEquals(0, configMapping.getReadQueueNums());
+ Assert.assertEquals(0, configMapping.getMappingDetail().getHostedQueues().size());
+ }
+
+ {
+ Set<String> targetBrokers = new HashSet<>();
+ targetBrokers.add(broker2Name);
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size());
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ }
+
+ }
+
+ private void sendMessagesAndCheck(RMQNormalProducer producer, Set<String> targetBrokers, String topic, int queueNum, int msgEachQueue, long baseOffset) throws Exception {
+ ClientMetadata clientMetadata = MQAdminUtils.getBrokerAndTopicMetadata(topic, defaultMQAdminExt);
+ List<MessageQueue> messageQueueList = producer.getMessageQueue();
+ Assert.assertEquals(queueNum, messageQueueList.size());
+ for (int i = 0; i < queueNum; i++) {
+ MessageQueue messageQueue = messageQueueList.get(i);
+ Assert.assertEquals(topic, messageQueue.getTopic());
+ Assert.assertEquals(TopicQueueMappingUtils.getMockBrokerName(MixAll.METADATA_SCOPE_GLOBAL), messageQueue.getBrokerName());
+ Assert.assertEquals(i, messageQueue.getQueueId());
+ String destBrokerName = clientMetadata.getBrokerNameFromMessageQueue(messageQueue);
+ Assert.assertTrue(targetBrokers.contains(destBrokerName));
+ }
+ for(MessageQueue messageQueue: messageQueueList) {
+ producer.send(msgEachQueue, messageQueue);
+ }
+ Assert.assertEquals(0, producer.getSendErrorMsg().size());
+ //leave the time to build the cq
+ Assert.assertTrue(awaitDispatchMs(500));
+ for(MessageQueue messageQueue: messageQueueList) {
+ Assert.assertEquals(0, defaultMQAdminExt.minOffset(messageQueue));
+ Assert.assertEquals(msgEachQueue + baseOffset, defaultMQAdminExt.maxOffset(messageQueue));
+ }
+ TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic);
+ for(MessageQueue messageQueue: messageQueueList) {
+ Assert.assertEquals(0, topicStatsTable.getOffsetTable().get(messageQueue).getMinOffset());
+ Assert.assertEquals(msgEachQueue + baseOffset, topicStatsTable.getOffsetTable().get(messageQueue).getMaxOffset());
+ }
+ }
+
+ private Map<Integer, List<MessageExt>> computeMessageByQueue(Collection<Object> msgs) {
+ Map<Integer, List<MessageExt>> messagesByQueue = new HashMap<>();
+ for (Object object : msgs) {
+ MessageExt messageExt = (MessageExt) object;
+ if (!messagesByQueue.containsKey(messageExt.getQueueId())) {
+ messagesByQueue.put(messageExt.getQueueId(), new ArrayList<>());
+ }
+ messagesByQueue.get(messageExt.getQueueId()).add(messageExt);
+ }
+ for (List<MessageExt> msgEachQueue: messagesByQueue.values()) {
+ Collections.sort(msgEachQueue, new Comparator<MessageExt>() {
+ @Override
+ public int compare(MessageExt o1, MessageExt o2) {
+ return (int) (o1.getQueueOffset() - o2.getQueueOffset());
+ }
+ });
+ }
+ return messagesByQueue;
+ }
+
+ private void consumeMessagesAndCheck(RMQNormalProducer producer, RMQNormalConsumer consumer, String topic, int queueNum, int msgEachQueue, int startGen, int genNum) {
+ consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), 30000);
- /*System.out.println("produce:" + producer.getAllMsgBody().size());
- System.out.println("consume:" + consumer.getListener().getAllMsgBody().size());*/
++// System.out.println("produce:" + producer.getAllMsgBody().size());
++// System.out.println("consume:" + consumer.getListener().getAllMsgBody().size());
+
+ Assert.assertEquals(producer.getAllMsgBody().size(), consumer.getListener().getAllMsgBody().size());
+ assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),
+ consumer.getListener().getAllMsgBody()))
+ .containsExactlyElementsIn(producer.getAllMsgBody());
+ Map<Integer, List<MessageExt>> messagesByQueue = computeMessageByQueue(consumer.getListener().getAllOriginMsg());
+ Assert.assertEquals(queueNum, messagesByQueue.size());
+ for (int i = 0; i < queueNum; i++) {
+ List<MessageExt> messageExts = messagesByQueue.get(i);
+ /*for (MessageExt messageExt:messageExts) {
+ System.out.printf("%d %d\n", messageExt.getQueueId(), messageExt.getQueueOffset());
+ }*/
+ int totalEachQueue = msgEachQueue * genNum;
+ Assert.assertEquals(totalEachQueue, messageExts.size());
+ for (int j = 0; j < totalEachQueue; j++) {
+ MessageExt messageExt = messageExts.get(j);
+ int currGen = startGen + j / msgEachQueue;
+ Assert.assertEquals(topic, messageExt.getTopic());
+ Assert.assertEquals(TopicQueueMappingUtils.getMockBrokerName(MixAll.METADATA_SCOPE_GLOBAL), messageExt.getBrokerName());
+ Assert.assertEquals(i, messageExt.getQueueId());
+ Assert.assertEquals((j % msgEachQueue) + currGen * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, messageExt.getQueueOffset());
+ }
+ }
+ }
+
+
+ @Test
+ public void testCreateProduceConsumeStaticTopic() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener());
+
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ //create static topic
+ Map<String, TopicConfigAndQueueMapping> localBrokerConfigMap = MQAdminTestUtils.createStaticTopic(topic, queueNum, getBrokers(), defaultMQAdminExt);
+ //check the static topic config
+ {
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size());
+ for (Map.Entry<String, TopicConfigAndQueueMapping> entry: remoteBrokerConfigMap.entrySet()) {
+ String broker = entry.getKey();
+ TopicConfigAndQueueMapping configMapping = entry.getValue();
+ TopicConfigAndQueueMapping localConfigMapping = localBrokerConfigMap.get(broker);
+ Assert.assertNotNull(localConfigMapping);
+ Assert.assertEquals(configMapping, localConfigMapping);
+ }
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ }
+ //send and check
+ sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 0);
+ //consume and check
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+
+
+ @Test
+ public void testRemappingProduceConsumeStaticTopic() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener());
+
+ int queueNum = 1;
+ int msgEachQueue = 100;
+ //create send consume
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+ //remapping the static topic
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker2Name);
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ for (TopicQueueMappingOne mappingOne: globalIdMap.values()) {
+ Assert.assertEquals(broker2Name, mappingOne.getBname());
+ Assert.assertEquals(TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, mappingOne.getItems().get(mappingOne.getItems().size() - 1).getLogicOffset());
+ }
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 2);
+ }
+ }
+
+
+ public boolean awaitRefreshStaticTopicMetadata(long timeMs, String topic, DefaultMQProducer producer, DefaultMQPushConsumer consumer, DefaultMQAdminExt adminExt) throws Exception {
+ long start = System.currentTimeMillis();
+ MQClientInstance currentInstance = null;
+ while (System.currentTimeMillis() - start <= timeMs) {
+ boolean allOk = true;
+ if (producer != null) {
+ currentInstance = producer.getDefaultMQProducerImpl().getmQClientFactory();
+ currentInstance.updateTopicRouteInfoFromNameServer(topic);
+ if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {
+ allOk = false;
+ }
+ }
+ if (consumer != null) {
+ currentInstance = consumer.getDefaultMQPushConsumerImpl().getmQClientFactory();
+ currentInstance.updateTopicRouteInfoFromNameServer(topic);
+ if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {
+ allOk = false;
+ }
+ }
+ if (adminExt != null) {
+ currentInstance = adminExt.getDefaultMQAdminExtImpl().getMqClientInstance();
+ currentInstance.updateTopicRouteInfoFromNameServer(topic);
+ if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {
+ allOk = false;
+ }
+ }
+ if (allOk) {
+ return true;
+ }
+ Thread.sleep(100);
+ }
+ return false;
+ }
+
+
+ @Test
+ public void testDoubleReadCheckConsumerOffset() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ String group = initConsumerGroup();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener());
+ long start = System.currentTimeMillis();
+
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ //create static topic
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+ producer.shutdown();
+ consumer.shutdown();
+ //use a new producer
+ producer = getProducer(nsAddr, topic);
+
+ ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(group);
+ List<MessageQueue> messageQueues = producer.getMessageQueue();
+ for (MessageQueue queue: messageQueues) {
+ OffsetWrapper wrapper = consumeStats.getOffsetTable().get(queue);
+ Assert.assertNotNull(wrapper);
+ Assert.assertEquals(msgEachQueue, wrapper.getBrokerOffset());
+ Assert.assertEquals(msgEachQueue, wrapper.getConsumerOffset());
+ Assert.assertTrue(wrapper.getLastTimestamp() > start);
+ }
+
+ List<String> brokers = ImmutableList.of(broker2Name, broker3Name, broker1Name);
+ for (int i = 0; i < brokers.size(); i++) {
+ Set<String> targetBrokers = ImmutableSet.of(brokers.get(i));
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ //make the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, (i + 1) * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ }
+
+ TestUtils.waitForSeconds(1);
+ consumeStats = defaultMQAdminExt.examineConsumeStats(group);
+
+ messageQueues = producer.getMessageQueue();
+ for (MessageQueue queue: messageQueues) {
+ OffsetWrapper wrapper = consumeStats.getOffsetTable().get(queue);
+ Assert.assertNotNull(wrapper);
+ Assert.assertEquals(msgEachQueue + brokers.size() * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, wrapper.getBrokerOffset());
+ Assert.assertEquals(msgEachQueue, wrapper.getConsumerOffset());
+ Assert.assertTrue(wrapper.getLastTimestamp() > start);
+ }
+ consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener());
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 1, brokers.size());
+ }
+
+
+ @Test
+ public void testRemappingAndClear() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ //create to broker1Name
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ //leave the time to refresh the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ }
+
+ //remapping to broker2Name
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker2Name);
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ //leave the time to refresh the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 1 * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ }
+
+ //remapping to broker3Name
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker3Name);
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ //leave the time to refresh the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 2 * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ }
+
+ // 1 -> 2 -> 3, currently 1 should not has any mappings
+
+ {
+ for (int i = 0; i < 10; i++) {
+ for (BrokerController brokerController: brokerControllerList) {
+ brokerController.getTopicQueueMappingCleanService().wakeup();
+ }
+ Thread.sleep(100);
+ }
+ Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, brokerConfigMap.size());
+ TopicConfigAndQueueMapping config1 = brokerConfigMap.get(broker1Name);
+ TopicConfigAndQueueMapping config2 = brokerConfigMap.get(broker2Name);
+ TopicConfigAndQueueMapping config3 = brokerConfigMap.get(broker3Name);
+ Assert.assertEquals(0, config1.getMappingDetail().getHostedQueues().size());
+ Assert.assertEquals(queueNum, config2.getMappingDetail().getHostedQueues().size());
+
+ Assert.assertEquals(queueNum, config3.getMappingDetail().getHostedQueues().size());
+
+ }
+ {
+ Set<String> topics = new HashSet<>(brokerController1.getTopicConfigManager().getTopicConfigTable().keySet());
+ topics.remove(topic);
+ brokerController1.getMessageStore().cleanUnusedTopic(topics);
+ brokerController2.getMessageStore().cleanUnusedTopic(topics);
+ for (int i = 0; i < 10; i++) {
+ for (BrokerController brokerController: brokerControllerList) {
+ brokerController.getTopicQueueMappingCleanService().wakeup();
+ }
+ Thread.sleep(100);
+ }
+
+ Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, brokerConfigMap.size());
+ TopicConfigAndQueueMapping config1 = brokerConfigMap.get(broker1Name);
+ TopicConfigAndQueueMapping config2 = brokerConfigMap.get(broker2Name);
+ TopicConfigAndQueueMapping config3 = brokerConfigMap.get(broker3Name);
+ Assert.assertEquals(0, config1.getMappingDetail().getHostedQueues().size());
+ Assert.assertEquals(queueNum, config2.getMappingDetail().getHostedQueues().size());
+ Assert.assertEquals(queueNum, config3.getMappingDetail().getHostedQueues().size());
+ //The first leader will clear it
+ for (List<LogicQueueMappingItem> items : config1.getMappingDetail().getHostedQueues().values()) {
+ Assert.assertEquals(3, items.size());
+ }
+ //The second leader do nothing
+ for (List<LogicQueueMappingItem> items : config3.getMappingDetail().getHostedQueues().values()) {
+ Assert.assertEquals(1, items.size());
+ }
+ }
+ }
+
+
+ @Test
+ public void testRemappingWithNegativeLogicOffset() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ //create and send
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ }
+
+ //remapping the static topic with -1 logic offset
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker2Name);
+ MQAdminTestUtils.remappingStaticTopicWithNegativeLogicOffset(topic, targetBrokers, defaultMQAdminExt);
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ for (TopicQueueMappingOne mappingOne: globalIdMap.values()) {
+ Assert.assertEquals(broker2Name, mappingOne.getBname());
+ Assert.assertEquals(-1, mappingOne.getItems().get(mappingOne.getItems().size() - 1).getLogicOffset());
+ }
+ //leave the time to refresh the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ //here the gen should be 0
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ }
+ }
+
+
+ @After
+ public void tearDown() {
+ System.setProperty("rocketmq.client.rebalance.waitInterval", "20000");
+ super.shutdown();
+ }
+
+ }
diff --cc tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
index 2206bc175,b8bb39979..311beb7f8
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
@@@ -41,9 -37,9 +37,10 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.body.ConsumerConnection;
import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.GroupList;
+ import org.apache.rocketmq.common.protocol.body.HARuntimeInfo;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
@@@ -291,11 -318,11 +324,16 @@@ public class DefaultMQAdminExt extends
return defaultMQAdminExtImpl.examineProducerConnectionInfo(producerGroup, topic);
}
+ @Override
+ public ProducerTableInfo getAllProducerInfo(final String brokerAddr) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
+ return defaultMQAdminExtImpl.getAllProducerInfo(brokerAddr);
+ }
+
+ @Override public void deleteTopicInNameServer(Set<String> addrs, String clusterName,
+ String topic) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+ defaultMQAdminExtImpl.deleteTopicInNameServer(addrs, clusterName, topic);
+ }
+
@Override
public List<String> getNameServerAddressList() {
return this.defaultMQAdminExtImpl.getNameServerAddressList();
diff --cc tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
index 9e99925a8,7b78ead3a..79f369b1e
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
@@@ -78,7 -88,11 +90,10 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+ import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+ import org.apache.rocketmq.common.subscription.GroupForbidden;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
-import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.common.RemotingUtil;
@@@ -213,16 -259,10 +260,15 @@@ public class DefaultMQAdminExtImpl impl
@Override public void updateGlobalWhiteAddrConfig(String addr,
String globalWhiteAddrs) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
- this.mqClientInstance.getMQClientAPIImpl().updateGlobalWhiteAddrsConfig(addr, globalWhiteAddrs, timeoutMillis);
+ this.mqClientInstance.getMQClientAPIImpl().updateGlobalWhiteAddrsConfig(addr, globalWhiteAddrs, null, timeoutMillis);
+ }
+
+ @Override public void updateGlobalWhiteAddrConfig(String addr,
+ String globalWhiteAddrs, String aclFileFullPath) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+ this.mqClientInstance.getMQClientAPIImpl().updateGlobalWhiteAddrsConfig(addr, globalWhiteAddrs, aclFileFullPath, timeoutMillis);
}
- @Override
- public ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
+ @Override public ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
String addr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
return this.mqClientInstance.getMQClientAPIImpl().getBrokerClusterAclInfo(addr, timeoutMillis);
}
@@@ -754,45 -1085,8 +1097,44 @@@
return result;
}
+ @Override
+ public boolean deleteExpiredCommitLog(String cluster) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException {
+ boolean result = false;
+ try {
+ ClusterInfo clusterInfo = examineBrokerClusterInfo();
+ if (null == cluster || "".equals(cluster)) {
+ for (String targetCluster : clusterInfo.retrieveAllClusterNames()) {
+ result = deleteExpiredCommitLogByCluster(clusterInfo, targetCluster);
+ }
+ } else {
+ result = deleteExpiredCommitLogByCluster(clusterInfo, cluster);
+ }
+ } catch (MQBrokerException e) {
+ log.error("deleteExpiredCommitLog error.", e);
+ }
+
+ return result;
+ }
+
+ public boolean deleteExpiredCommitLogByCluster(ClusterInfo clusterInfo, String cluster) throws RemotingConnectException,
+ RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException {
+ boolean result = false;
+ String[] addrs = clusterInfo.retrieveAllAddrByCluster(cluster);
+ for (String addr : addrs) {
+ result = deleteExpiredCommitLogByAddr(addr);
+ }
+ return result;
+ }
+
+ @Override
+ public boolean deleteExpiredCommitLogByAddr(String addr) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException {
+ boolean result = mqClientInstance.getMQClientAPIImpl().deleteExpiredCommitLog(addr, timeoutMillis);
+ log.warn("Delete expired CommitLog on target " + addr + " broker " + result);
+ return result;
+ }
+
- @Override
- public boolean cleanUnusedTopic(String cluster) throws RemotingConnectException, RemotingSendRequestException,
- RemotingTimeoutException, MQClientException, InterruptedException {
+ @Override public boolean cleanUnusedTopic(
+ String cluster) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException {
boolean result = false;
try {
ClusterInfo clusterInfo = examineBrokerClusterInfo();
diff --cc tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
index 58e3fb713,5ed147c13..59ac6c5cb
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
@@@ -40,9 -36,9 +36,10 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.body.ConsumerConnection;
import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.GroupList;
+ import org.apache.rocketmq.common.protocol.body.HARuntimeInfo;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
@@@ -78,13 -92,12 +93,15 @@@ public interface MQAdminExt extends MQA
void deletePlainAccessConfig(final String addr, final String accessKey) throws RemotingException, MQBrokerException,
InterruptedException, MQClientException;
- void updateGlobalWhiteAddrConfig(final String addr, final String globalWhiteAddrs)throws RemotingException, MQBrokerException,
+ void updateGlobalWhiteAddrConfig(final String addr,
+ final String globalWhiteAddrs) throws RemotingException, MQBrokerException,
InterruptedException, MQClientException;
+ void updateGlobalWhiteAddrConfig(final String addr, final String globalWhiteAddrs, String aclFileFullPath)throws RemotingException, MQBrokerException,
+ InterruptedException, MQClientException;
+
- ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(final String addr) throws RemotingException, MQBrokerException,
+ ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
+ final String addr) throws RemotingException, MQBrokerException,
InterruptedException, MQClientException;
AclConfig examineBrokerClusterAclConfig(final String addr) throws RemotingException, MQBrokerException,
diff --cc tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
index c69b7bf9e,bc720383b..dbf1a8eeb
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
@@@ -37,8 -37,8 +37,9 @@@ import org.apache.rocketmq.tools.comman
import org.apache.rocketmq.tools.command.broker.BrokerStatusSubCommand;
import org.apache.rocketmq.tools.command.broker.CleanExpiredCQSubCommand;
import org.apache.rocketmq.tools.command.broker.CleanUnusedTopicCommand;
+import org.apache.rocketmq.tools.command.broker.DeleteExpiredCommitLogSubCommand;
import org.apache.rocketmq.tools.command.broker.GetBrokerConfigCommand;
+ import org.apache.rocketmq.tools.command.broker.ResetMasterFlushOffsetSubCommand;
import org.apache.rocketmq.tools.command.broker.SendMsgStatusCommand;
import org.apache.rocketmq.tools.command.broker.UpdateBrokerConfigSubCommand;
import org.apache.rocketmq.tools.command.cluster.CLusterSendMsgRTCommand;
@@@ -49,11 -49,15 +50,15 @@@ import org.apache.rocketmq.tools.comman
import org.apache.rocketmq.tools.command.consumer.ConsumerStatusSubCommand;
import org.apache.rocketmq.tools.command.consumer.DeleteSubscriptionGroupCommand;
import org.apache.rocketmq.tools.command.consumer.GetConsumerConfigSubCommand;
+ import org.apache.rocketmq.tools.command.consumer.SetConsumeModeSubCommand;
import org.apache.rocketmq.tools.command.consumer.StartMonitoringSubCommand;
import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupSubCommand;
-import org.apache.rocketmq.tools.command.container.AddBrokerSubCommand;
-import org.apache.rocketmq.tools.command.container.RemoveBrokerSubCommand;
import org.apache.rocketmq.tools.command.export.ExportConfigsCommand;
import org.apache.rocketmq.tools.command.export.ExportMetadataCommand;
import org.apache.rocketmq.tools.command.export.ExportMetricsCommand;
++import org.apache.rocketmq.tools.command.container.AddBrokerSubCommand;
++import org.apache.rocketmq.tools.command.container.RemoveBrokerSubCommand;
+ import org.apache.rocketmq.tools.command.ha.HAStatusSubCommand;
import org.apache.rocketmq.tools.command.message.CheckMsgSendRTCommand;
import org.apache.rocketmq.tools.command.message.ConsumeMessageCommand;
import org.apache.rocketmq.tools.command.message.PrintMessageByQueueCommand;
diff --cc tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
index febc80e44,59df145eb..17e04e7f5
--- a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
+++ b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
@@@ -22,9 -34,10 +22,9 @@@ import org.apache.rocketmq.client.excep
import org.apache.rocketmq.client.impl.MQClientAPIImpl;
import org.apache.rocketmq.client.impl.MQClientManager;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.admin.ConsumeStats;
--import org.apache.rocketmq.common.admin.OffsetWrapper;
import org.apache.rocketmq.common.admin.TopicOffset;
import org.apache.rocketmq.common.admin.TopicStatsTable;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
@@@ -59,30 -71,18 +60,33 @@@ import org.apache.rocketmq.remoting.exc
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
import org.apache.rocketmq.tools.admin.api.MessageTrack;
+ import org.assertj.core.util.Maps;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+ import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.anyBoolean;
+ import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;