You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by ji...@apache.org on 2022/07/08 07:04:47 UTC
[rocketmq] branch develop-merge-5.0.0 created (now 3fadc8f30)
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a change to branch develop-merge-5.0.0
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
at 3fadc8f30 Merge branch '5.0.0-beta' into develop
This branch includes the following new commits:
new 3fadc8f30 Merge branch '5.0.0-beta' into develop
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[rocketmq] 01/01: Merge branch '5.0.0-beta' into develop
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch develop-merge-5.0.0
in repository https://gitbox.apache.org/repos/asf/rocketmq.git
commit 3fadc8f30c898f9ddd6730ba4ac428894a1c1077
Merge: 418a5b2ea ef37465e5
Author: RongtongJin <ji...@mails.ucas.ac.cn>
AuthorDate: Fri Jul 8 15:03:54 2022 +0800
Merge branch '5.0.0-beta' into develop
# Conflicts:
# acl/pom.xml
# broker/pom.xml
# broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
# broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
# broker/src/main/java/org/apache/rocketmq/broker/plugin/MessageStoreFactory.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/ForwardRequestProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
# broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java
# broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java
# client/pom.xml
# client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
# client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
# client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
# client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
# client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
# client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java
# common/pom.xml
# common/src/main/java/org/apache/rocketmq/common/MQVersion.java
# common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
# common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
# common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
# common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
# common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
# common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
# common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java
# distribution/pom.xml
# example/pom.xml
# filter/pom.xml
# logging/pom.xml
# logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java
# namesrv/pom.xml
# namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java
# namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
# namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
# openmessaging/pom.xml
# pom.xml
# remoting/pom.xml
# remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
# remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
# remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
# remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java
# srvutil/pom.xml
# store/pom.xml
# store/src/main/java/org/apache/rocketmq/store/CommitLog.java
# store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
# store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
# store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java
# store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
# store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java
# store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
# store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
# store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
# store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
# store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreCleanFilesTest.java
# store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
# store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java
# test/pom.xml
# tools/pom.xml
# tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
# tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
# tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
# tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
# tools/src/main/java/org/apache/rocketmq/tools/command/topic/DeleteTopicSubCommand.java
.travis.yml | 9 +-
acl/pom.xml | 2 +-
.../org/apache/rocketmq/acl/common/AclUtils.java | 39 +
.../apache/rocketmq/acl/common/AclUtilsTest.java | 21 +
acl/src/test/resources/conf/plain_acl.yml | 5 -
broker/pom.xml | 10 +-
.../apache/rocketmq/broker/BrokerController.java | 1761 ++++++++++++++-----
.../rocketmq/broker/BrokerPathConfigHelper.java | 12 +
.../rocketmq/broker/BrokerPreOnlineService.java | 277 +++
.../org/apache/rocketmq/broker/BrokerStartup.java | 58 +-
.../org/apache/rocketmq/broker/ShutdownHook.java | 15 +-
.../broker/client/ClientHousekeepingService.java | 12 +-
.../rocketmq/broker/client/ConsumerGroupInfo.java | 25 +-
.../broker/client/ConsumerIdsChangeListener.java | 2 +
.../rocketmq/broker/client/ConsumerManager.java | 42 +-
.../client/DefaultConsumerIdsChangeListener.java | 68 +-
.../rocketmq/broker/client/ProducerManager.java | 23 +-
.../rocketmq/broker/client/net/Broker2Client.java | 8 +-
.../client/rebalance/RebalanceLockManager.java | 99 +-
.../broker/dledger/DLedgerRoleChangeHandler.java | 90 +-
.../rocketmq/broker/failover/EscapeBridge.java | 268 +++
.../broker/filtersrv/FilterServerManager.java | 6 +-
.../rocketmq/broker/latency/BrokerFastFailure.java | 44 +-
.../broker/loadbalance/AssignmentManager.java | 140 ++
.../loadbalance/MessageRequestModeManager.java | 95 +
.../longpolling/LmqPullRequestHoldService.java | 3 +
.../broker/longpolling/ManyPullRequest.java | 4 +
.../broker/longpolling/NotificationRequest.java | 57 +
.../longpolling/NotifyMessageArrivingListener.java | 12 +-
.../rocketmq/broker/longpolling/PopRequest.java | 89 +
.../broker/longpolling/PullRequestHoldService.java | 36 +-
.../broker/mqtrace/AbortProcessException.java | 69 +
.../broker/mqtrace/ConsumeMessageContext.java | 77 +-
.../broker/mqtrace/SendMessageContext.java | 102 +-
.../broker/offset/ConsumerOffsetManager.java | 85 +-
.../broker/offset/ConsumerOrderInfoManager.java | 426 +++++
.../apache/rocketmq/broker/out/BrokerOuterAPI.java | 750 +++++++-
.../broker/pagecache/OneMessageTransfer.java | 1 +
.../broker/plugin/AbstractPluginMessageStore.java | 292 +++-
.../broker/plugin/BrokerAttachedPlugin.java | 74 +
.../broker/plugin/MessageStoreFactory.java | 15 +-
.../broker/plugin/MessageStorePluginContext.java | 16 +-
.../broker/plugin/PullMessageResultHandler.java | 53 +
.../processor/AbstractSendMessageProcessor.java | 457 ++++-
.../broker/processor/AckMessageProcessor.java | 214 +++
.../broker/processor/AdminBrokerProcessor.java | 992 +++++++++--
.../processor/ChangeInvisibleTimeProcessor.java | 206 +++
.../broker/processor/ClientManageProcessor.java | 52 +-
.../broker/processor/ConsumerManageProcessor.java | 188 +-
.../processor/DefaultPullMessageResultHandler.java | 244 +++
.../broker/processor/EndTransactionProcessor.java | 10 +-
.../broker/processor/ForwardRequestProcessor.java | 4 +-
.../broker/processor/NotificationProcessor.java | 318 ++++
.../broker/processor/PeekMessageProcessor.java | 271 +++
.../broker/processor/PollingInfoProcessor.java | 119 ++
.../broker/processor/PopBufferMergeService.java | 776 +++++++++
.../broker/processor/PopMessageProcessor.java | 983 +++++++++++
.../broker/processor/PopReviveService.java | 488 ++++++
.../broker/processor/PullMessageProcessor.java | 571 +++---
.../broker/processor/QueryAssignmentProcessor.java | 324 ++++
.../broker/processor/QueryMessageProcessor.java | 15 +-
.../broker/processor/ReplyMessageProcessor.java | 13 +-
.../broker/processor/SendMessageCallback.java | 22 +-
.../broker/processor/SendMessageProcessor.java | 603 +++----
.../schedule/DelayOffsetSerializeWrapper.java | 13 +-
.../broker}/schedule/ScheduleMessageService.java | 183 +-
.../rocketmq/broker/slave/SlaveSynchronize.java | 73 +-
.../subscription/SubscriptionGroupManager.java | 112 +-
.../rocketmq/broker/topic/TopicConfigManager.java | 287 ++-
.../topic/TopicQueueMappingCleanService.java | 337 ++++
.../broker/topic/TopicQueueMappingManager.java | 259 +++
.../AbstractTransactionalMessageCheckListener.java | 45 +-
.../broker/transaction/OperationResult.java | 8 +-
.../TransactionalMessageCheckService.java | 3 +
.../transaction/TransactionalMessageService.java | 2 +-
.../transaction/jdbc/JDBCTransactionStore.java | 242 ---
.../jdbc/JDBCTransactionStoreConfig.java | 57 -
.../DefaultTransactionalMessageCheckListener.java | 2 +-
.../queue/TransactionalMessageBridge.java | 2 +-
.../queue/TransactionalMessageServiceImpl.java | 2 +-
.../org/apache/rocketmq/broker/util/HookUtils.java | 164 ++
.../OperationResult.java => util/MsgUtil.java} | 35 +-
.../rocketmq/broker/BrokerControllerTest.java | 3 +-
.../apache/rocketmq/broker/BrokerOuterAPITest.java | 61 +-
.../apache/rocketmq/broker/BrokerStartupTest.java | 20 +-
.../broker/filter/MessageStoreWithFilterTest.java | 56 +-
.../broker/offset/ConsumerOffsetManagerTest.java | 55 +
.../broker/pagecache/OneMessageTransferTest.java | 8 +-
...essorTest.java => AckMessageProcessorTest.java} | 97 +-
.../broker/processor/AdminBrokerProcessorTest.java | 107 +-
....java => ChangeInvisibleTimeProcessorTest.java} | 96 +-
.../processor/ClientManageProcessorTest.java | 1 -
.../processor/ConsumerManageProcessorTest.java | 91 +
.../processor/EndTransactionProcessorTest.java | 2 +-
.../processor/PopBufferMergeServiceTest.java | 120 ++
...essorTest.java => PopMessageProcessorTest.java} | 163 +-
.../broker/processor/PullMessageProcessorTest.java | 28 +-
.../processor/QueryAssignmentProcessorTest.java | 227 +++
.../processor/ReplyMessageProcessorTest.java | 2 +-
.../broker/processor/SendMessageProcessorTest.java | 217 ++-
.../schedule/ScheduleMessageServiceTest.java | 145 +-
.../broker/substription/ForbiddenTest.java | 64 +
.../broker/topic/TopicConfigManagerTest.java | 324 ++++
.../broker/topic/TopicQueueMappingManagerTest.java | 112 ++
...faultTransactionalMessageCheckListenerTest.java | 2 +-
.../queue/TransactionalMessageBridgeTest.java | 2 +-
.../queue/TransactionalMessageServiceImplTest.java | 2 +-
.../rocketmq/broker/util/ServiceProviderTest.java | 7 +-
.../util/TransactionalMessageServiceImpl.java | 2 +-
client/pom.xml | 11 +-
.../java/org/apache/rocketmq/client/MQAdmin.java | 14 +-
.../rocketmq/client/consumer/AckCallback.java | 10 +-
.../apache/rocketmq/client/consumer/AckResult.java | 40 +-
.../apache/rocketmq/client/consumer/AckStatus.java | 17 +-
.../client/consumer/DefaultMQPullConsumer.java | 14 +-
.../client/consumer/DefaultMQPushConsumer.java | 70 +-
.../rocketmq/client/consumer/MQPullConsumer.java | 7 +
.../rocketmq/client/consumer/PopCallback.java | 13 +-
.../consumer/{PullResult.java => PopResult.java} | 59 +-
.../apache/rocketmq/client/consumer/PopStatus.java | 29 +-
.../rocketmq/client/consumer/PullResult.java | 1 +
.../AbstractAllocateMessageQueueStrategy.java | 2 +-
.../rebalance/AllocateMessageQueueAveragely.java | 10 +
.../AllocateMessageQueueAveragelyByCircle.java | 10 +
.../consumer/store/RemoteBrokerOffsetStore.java | 31 +-
.../client/exception/MQBrokerException.java | 6 +
.../client/exception/MQRedirectException.java | 26 +-
.../client/exception/OffsetNotFoundException.java | 23 +-
.../rocketmq/client/impl/BaseInvokeCallback.java | 26 +-
.../client/impl/ClientRemotingProcessor.java | 3 +-
.../apache/rocketmq/client/impl/MQAdminImpl.java | 64 +-
.../rocketmq/client/impl/MQClientAPIImpl.java | 632 ++++++-
.../ConsumeMessageConcurrentlyService.java | 11 +-
.../consumer/ConsumeMessageOrderlyService.java | 14 +-
...a => ConsumeMessagePopConcurrentlyService.java} | 260 +--
.../consumer/ConsumeMessagePopOrderlyService.java | 408 +++++
.../impl/consumer/ConsumeMessageService.java | 5 +
.../impl/consumer/DefaultLitePullConsumerImpl.java | 45 +-
.../impl/consumer/DefaultMQPullConsumerImpl.java | 43 +-
.../impl/consumer/DefaultMQPushConsumerImpl.java | 412 ++++-
.../client/impl/consumer/MessageQueueLock.java | 29 +-
.../client/impl/consumer/MessageRequest.java | 10 +-
.../client/impl/consumer/PopProcessQueue.java | 84 +
.../consumer/{PullRequest.java => PopRequest.java} | 71 +-
.../client/impl/consumer/PullAPIWrapper.java | 141 +-
.../client/impl/consumer/PullMessageService.java | 47 +-
.../rocketmq/client/impl/consumer/PullRequest.java | 8 +-
.../client/impl/consumer/PullResultExt.java | 12 +
.../client/impl/consumer/RebalanceImpl.java | 479 ++++-
.../impl/consumer/RebalanceLitePullImpl.java | 25 +-
.../client/impl/consumer/RebalancePullImpl.java | 26 +-
.../client/impl/consumer/RebalancePushImpl.java | 71 +-
.../client/impl/factory/MQClientInstance.java | 174 +-
.../impl/producer/DefaultMQProducerImpl.java | 20 +-
.../rocketmq/client/latency/MQFaultStrategy.java | 2 +-
.../client/producer/DefaultMQProducer.java | 16 +-
.../rocketmq/client/producer/SendResult.java | 9 +
.../consumer/DefaultLitePullConsumerTest.java | 32 +-
.../client/consumer/DefaultMQPushConsumerTest.java | 83 +-
.../store/RemoteBrokerOffsetStoreTest.java | 10 +-
.../rocketmq/client/impl/MQClientAPIImplTest.java | 463 ++++-
.../consumer/DefaultMQPushConsumerImplTest.java | 63 +
.../impl/consumer/RebalancePushImplTest.java | 13 -
.../client/impl/factory/MQClientInstanceTest.java | 1 +
.../client/producer/DefaultMQProducerTest.java | 2 +-
.../selector/SelectMessageQueueRetryTest.java | 1 -
.../powermock/extensions/configuration.properties | 16 +
common/pom.xml | 6 +-
...uestHeader.java => AbstractBrokerRunnable.java} | 31 +-
.../org/apache/rocketmq/common/BrokerConfig.java | 621 ++++++-
.../org/apache/rocketmq/common/BrokerIdentity.java | 149 ++
.../org/apache/rocketmq/common/BrokerSyncInfo.java | 70 +
.../org/apache/rocketmq/common/ConfigManager.java | 11 +
.../org/apache/rocketmq/common/Configuration.java | 41 +
.../org/apache/rocketmq/common/DataVersion.java | 49 +-
.../{ThreadFactoryImpl.java => KeyBuilder.java} | 32 +-
.../org/apache/rocketmq/common/LockCallback.java | 12 +-
.../java/org/apache/rocketmq/common/MQVersion.java | 2 +-
.../java/org/apache/rocketmq/common/MixAll.java | 25 +-
.../apache/rocketmq/common/PopAckConstants.java | 44 +
.../org/apache/rocketmq/common/ServiceThread.java | 8 +-
.../apache/rocketmq/common/ThreadFactoryImpl.java | 13 +
.../apache/rocketmq/common/TopicAttributes.java | 47 +
.../org/apache/rocketmq/common/TopicConfig.java | 122 +-
.../org/apache/rocketmq/common/TopicQueueId.java | 54 +
.../MessageType.java => UnlockCallback.java} | 10 +-
.../java/org/apache/rocketmq/common/UtilAll.java | 129 +-
.../apache/rocketmq/common/admin/TopicOffset.java | 9 +
.../rocketmq/common/attribute/Attribute.java | 36 +-
.../rocketmq/common/attribute/AttributeParser.java | 79 +
.../BooleanAttribute.java} | 28 +-
.../MessageType.java => attribute/CQType.java} | 10 +-
.../EnumAttribute.java} | 30 +-
.../LongRangeAttribute.java} | 33 +-
.../TopicMessageType.java} | 29 +-
.../ConsumeInitMode.java} | 11 +-
.../rocketmq/common/constant/LoggerName.java | 9 +-
.../apache/rocketmq/common/constant/PermName.java | 18 +-
.../fastjson/GenericMapSuperclassDeserializer.java | 58 +
.../rocketmq/common/future/FutureTaskExt.java | 31 +-
.../rocketmq/common/message/MessageConst.java | 34 +-
.../rocketmq/common/message/MessageDecoder.java | 201 ++-
.../apache/rocketmq/common/message/MessageExt.java | 55 +
.../rocketmq/common/message/MessageExtBatch.java | 15 +-
.../common/message}/MessageExtBrokerInner.java | 3 +-
.../rocketmq/common/message/MessageQueue.java | 6 +
.../common/message/MessageQueueAssignment.java | 83 +
.../{MessageType.java => MessageRequestMode.java} | 30 +-
.../rocketmq/common/message/MessageType.java | 28 +-
.../rocketmq/common/message/MessageVersion.java | 70 +
.../common/namesrv/DefaultTopAddressing.java | 165 ++
.../NameServerUpdateCallback.java} | 10 +-
.../rocketmq/common/namesrv/NamesrvConfig.java | 134 +-
.../rocketmq/common/namesrv/TopAddressing.java | 88 +-
.../rocketmq/common/protocol/ForbiddenType.java | 38 +-
.../rocketmq/common/protocol/RequestCode.java | 42 +
.../rocketmq/common/protocol/ResponseCode.java | 18 +
.../common/protocol/body/BrokerMemberGroup.java | 90 +
.../protocol/body/CheckClientRequestBody.java | 9 +
.../rocketmq/common/protocol/body/ClusterInfo.java | 28 +-
.../body/ConsumerOffsetSerializeWrapper.java | 10 +
.../common/protocol/body/ConsumerRunningInfo.java | 63 +-
.../body/GetBrokerMemberGroupResponseBody.java | 19 +-
.../protocol/body/GetRemoteClientConfigBody.java | 26 +-
.../common/protocol/body/HARuntimeInfo.java | 188 ++
.../common/protocol/body/LockBatchRequestBody.java | 9 +
...ava => MessageRequestModeSerializeWrapper.java} | 20 +-
...ntRequestBody.java => PopProcessQueueInfo.java} | 45 +-
...stBody.java => QueryAssignmentRequestBody.java} | 39 +-
.../protocol/body/QueryAssignmentResponseBody.java | 22 +-
...ody.java => QuerySubscriptionResponseBody.java} | 22 +-
.../common/protocol/body/RegisterBrokerBody.java | 32 +-
.../SetMessageRequestModeRequestBody.java} | 54 +-
.../TopicConfigAndMappingSerializeWrapper.java | 68 +
...java => TopicQueueMappingSerializeWrapper.java} | 35 +-
.../protocol/body/UnlockBatchRequestBody.java | 9 +
...estHeader.java => AckMessageRequestHeader.java} | 37 +-
...uestHeader.java => AddBrokerRequestHeader.java} | 21 +-
....java => ChangeInvisibleTimeRequestHeader.java} | 51 +-
...java => ChangeInvisibleTimeResponseHeader.java} | 39 +-
.../ConsumeMessageDirectlyResultRequestHeader.java | 30 +
.../protocol/header/CreateTopicRequestHeader.java | 21 +
.../DeleteSubscriptionGroupRequestHeader.java | 10 +-
...eader.java => ExchangeHAInfoRequestHeader.java} | 44 +-
...ader.java => ExchangeHAInfoResponseHeader.java} | 44 +-
.../common/protocol/header/ExtraInfoUtil.java | 258 +++
...java => GetBrokerMemberGroupRequestHeader.java} | 31 +-
.../GetEarliestMsgStoretimeRequestHeader.java | 8 +-
.../protocol/header/GetMaxOffsetRequestHeader.java | 26 +-
.../protocol/header/GetMinOffsetRequestHeader.java | 8 +-
...> GetSubscriptionGroupConfigRequestHeader.java} | 24 +-
...eader.java => GetTopicConfigRequestHeader.java} | 19 +-
.../header/GetTopicStatsInfoRequestHeader.java | 4 +-
...r.java => InitConsumerOffsetRequestHeader.java} | 17 +-
...tHeader.java => NotificationRequestHeader.java} | 44 +-
...Header.java => NotificationResponseHeader.java} | 19 +-
...a => NotifyMinBrokerIdChangeRequestHeader.java} | 52 +-
...stHeader.java => PeekMessageRequestHeader.java} | 28 +-
...stHeader.java => PollingInfoRequestHeader.java} | 16 +-
...tHeader.java => PollingInfoResponseHeader.java} | 18 +-
.../protocol/header/PopMessageRequestHeader.java | 155 ++
.../protocol/header/PopMessageResponseHeader.java | 102 ++
.../protocol/header/PullMessageRequestHeader.java | 56 +-
.../protocol/header/PullMessageResponseHeader.java | 66 +
.../header/QueryConsumerOffsetRequestHeader.java | 18 +-
... QuerySubscriptionByConsumerRequestHeader.java} | 24 +-
...ava => QueryTopicsByConsumerRequestHeader.java} | 16 +-
...tHeader.java => RemoveBrokerRequestHeader.java} | 39 +-
...ader.java => ResetMasterFlushOffsetHeader.java} | 13 +-
.../protocol/header/SearchOffsetRequestHeader.java | 8 +-
.../protocol/header/SendMessageRequestHeader.java | 8 +-
.../protocol/header/SendMessageResponseHeader.java | 15 +
...r.java => StatisticsMessagesRequestHeader.java} | 37 +-
.../header/UpdateConsumerOffsetRequestHeader.java | 8 +-
...java => UpdateGroupForbiddenRequestHeader.java} | 28 +-
...ader.java => BrokerHeartbeatRequestHeader.java} | 58 +-
.../header/namesrv/GetRouteInfoRequestHeader.java | 2 +
.../namesrv/RegisterBrokerRequestHeader.java | 22 +
.../RegisterTopicRequestHeader.java} | 5 +-
.../rocketmq/common/protocol/route/BrokerData.java | 55 +-
.../route/MessageQueueRouteState.java} | 15 +-
.../rocketmq/common/protocol/route/QueueData.java | 13 +
.../common/protocol/route/TopicRouteData.java | 92 +-
.../common/protocol/route/TopicRouteDatas.java | 21 +-
.../apache/rocketmq/common/rpc/ClientMetadata.java | 176 ++
.../apache/rocketmq/common/rpc/RequestBuilder.java | 81 +
.../org/apache/rocketmq/common/rpc/RpcClient.java | 29 +-
.../apache/rocketmq/common/rpc/RpcClientHook.java | 12 +-
.../apache/rocketmq/common/rpc/RpcClientImpl.java | 340 ++++
.../apache/rocketmq/common/rpc/RpcClientUtils.java | 58 +
.../apache/rocketmq/common/rpc/RpcException.java | 31 +-
.../org/apache/rocketmq/common/rpc/RpcRequest.java | 33 +-
.../RpcRequestHeader.java} | 54 +-
.../apache/rocketmq/common/rpc/RpcResponse.java | 70 +
.../common/rpc/TopicQueueRequestHeader.java | 10 +-
.../rocketmq/common/rpc/TopicRequestHeader.java | 19 +-
.../common/statictopic/LogicQueueMappingItem.java | 212 +++
.../statictopic/TopicConfigAndQueueMapping.java | 63 +
.../statictopic/TopicQueueMappingContext.java | 99 ++
.../statictopic/TopicQueueMappingDetail.java | 144 ++
.../common/statictopic/TopicQueueMappingInfo.java | 161 ++
.../common/statictopic/TopicQueueMappingOne.java | 85 +
.../common/statictopic/TopicQueueMappingUtils.java | 694 ++++++++
.../statictopic/TopicRemappingDetailWrapper.java | 104 ++
.../rocketmq/common/statistics/FutureHolder.java | 53 +
.../Interceptor.java} | 18 +-
.../common/statistics/StatisticsBrief.java | 184 ++
.../statistics/StatisticsBriefInterceptor.java | 76 +
.../rocketmq/common/statistics/StatisticsItem.java | 175 ++
.../StatisticsItemFormatter.java} | 32 +-
.../common/statistics/StatisticsItemPrinter.java | 50 +
.../StatisticsItemScheduledIncrementPrinter.java | 290 +++
.../statistics/StatisticsItemScheduledPrinter.java | 97 ++
.../StatisticsItemStateGetter.java} | 10 +-
.../StatisticsKindMeta.java} | 39 +-
.../common/statistics/StatisticsManager.java | 157 ++
.../common/subscription/CustomizedRetryPolicy.java | 85 +
.../subscription/ExponentialRetryPolicy.java | 74 +
.../common/subscription/GroupForbidden.java | 86 +
.../common/subscription/GroupRetryPolicy.java | 78 +
.../GroupRetryPolicyType.java} | 12 +-
.../rocketmq/common/subscription/RetryPolicy.java | 17 +-
.../subscription/SubscriptionGroupConfig.java | 95 +-
.../rocketmq/common/sysflag/MessageSysFlag.java | 5 +-
.../rocketmq/common/sysflag/PullSysFlag.java | 4 +
.../rocketmq/common/topic/TopicValidator.java | 2 +
.../rocketmq/common/utils/DataConverter.java | 35 +-
.../apache/rocketmq/common/utils/MessageUtils.java | 49 +
.../common/utils/PositiveAtomicCounter.java | 32 +-
.../rocketmq/common/utils/QueueTypeUtils.java | 51 +
.../rocketmq/common/utils}/ServiceProvider.java | 27 +-
.../apache/rocketmq/common/utils/ThreadUtils.java | 17 +-
.../apache/rocketmq/common/ConfigManagerTest.java | 5 +-
.../apache/rocketmq/common/DataVersionTest.java | 7 +
.../rocketmq/common/RegisterBrokerBodyTest.java | 5 +-
.../apache/rocketmq/common/TopicConfigTest.java | 78 +
.../org/apache/rocketmq/common/UtilAllTest.java | 21 +-
.../common/attribute/AttributeParserTest.java | 70 +
.../rocketmq/common/attribute/AttributeTest.java | 70 +
.../common/message/MessageClientIDSetterTest.java | 2 -
.../common/message/MessageDecoderTest.java | 12 +-
.../rocketmq/common/message/MessageTest.java | 1 -
.../rocketmq/common/protocol/body/KVTableTest.java | 1 -
.../MessageRequestModeSerializeWrapperTest.java | 58 +
.../common/protocol/route/TopicRouteDataTest.java | 11 +-
.../common/statictopic/TopicQueueMappingTest.java | 78 +
.../statictopic/TopicQueueMappingUtilsTest.java | 320 ++++
.../subscription/CustomizedRetryPolicyTest.java | 44 +
.../subscription/ExponentialRetryPolicyTest.java | 44 +
.../common/subscription/GroupRetryPolicyTest.java | 49 +
{logging => container}/pom.xml | 23 +-
.../apache/rocketmq/container/BrokerBootHook.java | 36 +-
.../apache/rocketmq/container/BrokerContainer.java | 478 +++++
.../rocketmq/container/BrokerContainerConfig.java | 75 +
.../container/BrokerContainerProcessor.java | 280 +++
.../rocketmq/container/BrokerContainerStartup.java | 445 +++++
.../ContainerClientHouseKeepingService.java | 104 ++
.../rocketmq/container/IBrokerContainer.java | 142 ++
.../rocketmq/container/InnerBrokerController.java | 204 +++
.../container/InnerSalveBrokerController.java | 46 +
.../logback/BrokerLogbackConfigurator.java | 187 ++
.../container/BrokerContainerStartupTest.java | 140 ++
.../rocketmq/container/BrokerContainerTest.java | 371 ++++
.../rocketmq/container/BrokerPreOnlineTest.java | 102 ++
distribution/bin/{mqshutdown => mqbrokercontainer} | 46 +-
distribution/bin/mqshutdown | 14 +
distribution/bin/runbroker.sh | 2 +-
.../2container-2m-2s/broker-a-in-container1.conf | 49 +-
.../2container-2m-2s/broker-a-in-container2.conf | 49 +-
.../2container-2m-2s/broker-b-in-container1.conf | 49 +-
.../2container-2m-2s/broker-b-in-container2.conf | 49 +-
.../2container-2m-2s/broker-container1.conf | 38 +-
.../2container-2m-2s/broker-container2.conf | 38 +-
.../container/2container-2m-2s/nameserver.conf | 30 +-
.../conf/container/broker-a.conf | 48 +-
.../conf/container/broker-b.conf | 48 +-
.../conf/container/broker-container.conf | 38 +-
distribution/conf/logback_broker.xml | 29 +
distribution/conf/logback_tools.xml | 5 +
distribution/pom.xml | 6 +-
distribution/release.xml | 1 +
docs/cn/BrokerContainer.md | 152 ++
docs/cn/QuorumACK.md | 70 +
docs/cn/README.md | 12 +-
docs/cn/SlaveActingMasterMode.md | 164 ++
..._Topic_Logic_Queue_\350\256\276\350\256\241.md" | 503 ++++++
docs/cn/statictopic/The_Scope_Of_Static_Topic.md | 116 ++
docs/en/Feature.md | 7 +-
example/pom.xml | 6 +-
.../rocketmq/example/benchmark/Consumer.java | 2 +
.../rocketmq/example/simple/PopPushConsumer.java | 62 +
filter/pom.xml | 2 +-
logging/pom.xml | 2 +-
.../rocketmq/logging/InternalLoggerFactory.java | 11 +
.../rocketmq/logging/Slf4jLoggerFactory.java | 87 +-
.../apache/rocketmq/logging/inner/SysLogger.java | 4 +-
.../rocketmq/logging/Slf4jLoggerFactoryTest.java | 2 +-
.../rocketmq/logging/inner/LoggingBuilderTest.java | 8 +-
namesrv/pom.xml | 14 +-
.../apache/rocketmq/namesrv/NamesrvController.java | 149 +-
.../apache/rocketmq/namesrv/NamesrvStartup.java | 11 +-
.../namesrv/processor/ClientRequestProcessor.java | 102 ++
.../processor/ClusterTestRequestProcessor.java | 2 +-
.../namesrv/processor/DefaultRequestProcessor.java | 364 ++--
.../namesrv/routeinfo/BatchUnRegisterService.java | 85 +
.../routeinfo/BrokerHousekeepingService.java | 6 +-
.../namesrv/routeinfo/RouteInfoManager.java | 1027 ++++++++---
...rocessorTest.java => RequestProcessorTest.java} | 159 +-
.../namesrv/routeinfo/GetRouteInfoBenchmark.java | 148 ++
.../namesrv/routeinfo/RegisterBrokerBenchmark.java | 177 ++
.../routeinfo/RouteInfoManagerBrokerPermTest.java | 24 +-
.../RouteInfoManagerBrokerRegisterTest.java | 61 +-
.../RouteInfoManagerStaticRegisterTest.java | 19 +-
.../namesrv/routeinfo/RouteInfoManagerTest.java | 223 +++
.../routeinfo/RouteInfoManagerTestBase.java | 1 +
.../routeinfo/RouteInfoManager_NewTest.java | 783 +++++++++
openmessaging/pom.xml | 2 +-
pom.xml | 22 +-
remoting/pom.xml | 2 +-
.../java/org/apache/rocketmq/remoting/RPCHook.java | 2 +-
.../apache/rocketmq/remoting/RemotingClient.java | 13 +-
.../apache/rocketmq/remoting/RemotingServer.java | 6 +
.../apache/rocketmq/remoting/RemotingService.java | 5 +
.../rocketmq/remoting/common/RemotingHelper.java | 39 +
.../rocketmq/remoting/common/RemotingUtil.java | 18 +-
.../rocketmq/remoting/netty/NettyClientConfig.java | 38 +
.../remoting/netty/NettyRemotingAbstract.java | 103 +-
.../remoting/netty/NettyRemotingClient.java | 392 ++++-
.../remoting/netty/NettyRemotingServer.java | 158 +-
.../remoting/netty/NettyRequestProcessor.java | 1 -
.../rocketmq/remoting/netty/ResponseFuture.java | 42 +-
.../remoting/protocol/FastCodesHeader.java | 2 +-
.../remoting/protocol/RemotingCommand.java | 54 +-
.../remoting/protocol/RemotingSerializable.java | 7 +-
.../rocketmq/remoting/RemotingServerTest.java | 5 +-
.../rocketmq/remoting/SubRemotingServerTest.java | 109 ++
.../remoting/netty/NettyRemotingAbstractTest.java | 6 +-
.../remoting/protocol/RemotingCommandTest.java | 99 +-
srvutil/pom.xml | 10 +-
.../rocketmq/srvutil/ConcurrentHashMapUtil.java | 50 +
.../apache/rocketmq/srvutil/FileWatchService.java | 4 +-
.../rocketmq/util/cache/CacheEvictHandler.java | 10 +-
.../apache/rocketmq/util/cache/CacheObject.java | 28 +-
.../rocketmq/util/cache/ExpiredLocalCache.java | 84 +
.../org/apache/rocketmq/util/cache/LocalCache.java | 58 +
.../apache/rocketmq/util/cache/LockManager.java | 54 +
store/pom.xml | 2 +-
.../rocketmq/store/AllocateMappedFileService.java | 11 +-
.../rocketmq/store/AppendMessageCallback.java | 2 +-
.../apache/rocketmq/store/AppendMessageResult.java | 12 +
.../java/org/apache/rocketmq/store/CommitLog.java | 1138 ++++++++----
.../org/apache/rocketmq/store/ConsumeQueue.java | 313 +++-
.../org/apache/rocketmq/store/ConsumeQueueExt.java | 5 +
.../apache/rocketmq/store/DefaultMessageStore.java | 1842 ++++++++++++--------
.../org/apache/rocketmq/store/DispatchRequest.java | 45 +
.../apache/rocketmq/store/FileQueueSnapshot.java | 90 +
.../apache/rocketmq/store/GetMessageResult.java | 37 +-
.../org/apache/rocketmq/store/MappedFileQueue.java | 189 +-
.../org/apache/rocketmq/store/MessageStore.java | 422 ++++-
.../org/apache/rocketmq/store/MultiDispatch.java | 184 --
.../rocketmq/store/MultiPathMappedFileQueue.java | 6 +-
.../apache/rocketmq/store/PutMessageContext.java | 48 +
.../apache/rocketmq/store/PutMessageResult.java | 26 +-
.../apache/rocketmq/store/PutMessageStatus.java | 6 +-
.../apache/rocketmq/store/QueryMessageResult.java | 4 +
.../rocketmq/store/SelectMappedBufferResult.java | 11 +-
.../rocketmq/store/SelectMappedFileResult.java | 34 +-
.../org/apache/rocketmq/store/StoreCheckpoint.java | 21 +-
.../apache/rocketmq/store/StoreStatsService.java | 11 +
.../java/org/apache/rocketmq/store/StoreUtil.java | 44 +
.../java/org/apache/rocketmq/store/Swappable.java | 14 +-
.../org/apache/rocketmq/store/TopicQueueLock.java | 46 +
.../rocketmq/store/config/MessageStoreConfig.java | 484 ++++-
.../store/config/StorePathConfigHelper.java | 3 +
.../rocketmq/store/dledger/DLedgerCommitLog.java | 275 ++-
.../apache/rocketmq/store/ha/DefaultHAClient.java | 392 +++++
...{HAConnection.java => DefaultHAConnection.java} | 187 +-
.../apache/rocketmq/store/ha/DefaultHAService.java | 339 ++++
.../org/apache/rocketmq/store/ha/FlowMonitor.java | 76 +
.../rocketmq/store/ha/GroupTransferService.java | 140 ++
.../org/apache/rocketmq/store/ha/HAClient.java | 104 ++
.../org/apache/rocketmq/store/ha/HAConnection.java | 428 +----
.../rocketmq/store/ha/HAConnectionState.java | 31 +-
.../ha/HAConnectionStateNotificationRequest.java | 49 +
.../ha/HAConnectionStateNotificationService.java | 150 ++
.../org/apache/rocketmq/store/ha/HAService.java | 665 +------
.../apache/rocketmq/store/ha/WaitNotifyObject.java | 67 +-
.../apache/rocketmq/store/hook/PutMessageHook.java | 25 +-
.../rocketmq/store/hook/SendMessageBackHook.java | 25 +-
.../org/apache/rocketmq/store/index/IndexFile.java | 21 +-
.../apache/rocketmq/store/index/IndexService.java | 75 +-
.../rocketmq/store/logfile/AbstractMappedFile.java | 9 +-
.../DefaultMappedFile.java} | 302 ++--
.../apache/rocketmq/store/logfile/MappedFile.java | 340 ++++
.../java/org/apache/rocketmq/store/pop/AckMsg.java | 97 ++
.../apache/rocketmq/store/pop/PopCheckPoint.java | 184 ++
.../rocketmq/store/queue/BatchConsumeQueue.java | 980 +++++++++++
.../rocketmq/store/queue/BatchOffsetIndex.java | 57 +
.../store/queue/ConsumeQueueInterface.java | 142 ++
.../rocketmq/store/queue/ConsumeQueueStore.java | 483 +++++
.../org/apache/rocketmq/store/queue/CqUnit.java | 115 ++
.../rocketmq/store/queue/FileQueueLifeCycle.java | 84 +
.../rocketmq/store/queue/QueueOffsetAssigner.java | 92 +
.../rocketmq/store/queue/ReferredIterator.java | 14 +-
.../apache/rocketmq/store/stats/BrokerStats.java | 6 +-
.../rocketmq/store/stats/BrokerStatsManager.java | 435 ++++-
.../apache/rocketmq/store/util/PerfCounter.java | 370 ++++
.../apache/rocketmq/store/AppendCallbackTest.java | 4 +-
.../apache/rocketmq/store/BatchPutMessageTest.java | 61 +-
.../apache/rocketmq/store/ConsumeQueueTest.java | 80 +-
.../store/DefaultMessageStoreCleanFilesTest.java | 92 +-
.../store/DefaultMessageStoreShutDownTest.java | 2 +-
.../rocketmq/store/DefaultMessageStoreTest.java | 285 ++-
.../java/org/apache/rocketmq/store/HATest.java | 130 +-
.../apache/rocketmq/store/MappedFileQueueTest.java | 156 +-
.../org/apache/rocketmq/store/MappedFileTest.java | 4 +-
.../apache/rocketmq/store/MultiDispatchTest.java | 63 +-
.../store/MultiPathMappedFileQueueTest.java | 32 +-
.../rocketmq/store/ScheduleMessageServiceTest.java | 194 ---
.../org/apache/rocketmq/store/StoreTestBase.java | 13 +-
.../store/dledger/DLedgerCommitlogTest.java | 20 +-
.../store/dledger/MessageStoreTestBase.java | 5 +-
.../rocketmq/store/dledger/MixCommitlogTest.java | 2 -
.../apache/rocketmq/store/ha/FlowMonitorTest.java | 61 +
.../org/apache/rocketmq/store/ha/HAClientTest.java | 72 +
.../org/apache/rocketmq/store/ha/HAServerTest.java | 295 ++++
.../rocketmq/store/ha/WaitNotifyObjectTest.java | 2 -
.../store/queue/BatchConsumeMessageTest.java | 454 +++++
.../store/queue/BatchConsumeQueueTest.java | 312 ++++
.../store/queue/ConsumeQueueStoreTest.java | 100 ++
.../rocketmq/store/queue/ConsumeQueueTest.java | 102 ++
.../apache/rocketmq/store/queue/QueueTestBase.java | 114 ++
test/pom.xml | 6 +-
.../test/client/rmq/RMQNormalConsumer.java | 5 +
.../test/client/rmq/RMQNormalProducer.java | 7 +
.../rocketmq/test/client/rmq/RMQPopConsumer.java | 33 +
.../test/clientinterface/AbstractMQProducer.java | 1 +
.../rocketmq/test/factory/ConsumerFactory.java | 10 +
.../rocketmq/test/listener/AbstractListener.java | 2 +-
.../org/apache/rocketmq/test/util/MQAdmin.java | 166 --
.../rocketmq/test/util/MQAdminTestUtils.java | 310 ++++
.../org/apache/rocketmq/test/base/BaseConf.java | 168 +-
.../rocketmq/test/base/IntegrationTestBase.java | 40 +-
.../base/dledger/DLedgerProduceAndConsumeIT.java | 3 +-
.../consumer/balance/NormalMsgStaticBalanceIT.java | 18 +-
.../normal/BroadCastNormalMsgRecvFailIT.java | 2 +
.../test/client/consumer/pop/PopSubCheckIT.java | 96 +
.../test/client/producer/batch/BatchSendIT.java | 161 ++
.../client/producer/oneway/OneWaySendWithMQIT.java | 10 -
.../client/producer/order/OrderMsgRebalanceIT.java | 2 +-
.../test/container/AddAndRemoveBrokerIT.java | 83 +
.../rocketmq/test/container/BrokerFailoverIT.java | 86 +
.../test/container/BrokerMemberGroupIT.java | 71 +
.../container/ContainerIntegrationTestBase.java | 666 +++++++
.../test/container/GetMaxOffsetFromSlaveIT.java | 100 ++
.../test/container/GetMetadataReverseIT.java | 231 +++
.../test/container/PullMultipleReplicasIT.java | 201 +++
.../test/container/PushMultipleReplicasIT.java | 114 ++
.../test/container/RebalanceLockOnSlaveIT.java | 209 +++
.../container/ScheduleSlaveActingMasterIT.java | 220 +++
.../test/container/ScheduledMessageIT.java | 153 ++
.../test/container/SendMultipleReplicasIT.java | 159 ++
.../rocketmq/test/container/SlaveBrokerIT.java | 117 ++
.../test/container/SyncConsumerOffsetIT.java | 148 ++
.../rocketmq/test/delay/NormalMsgDelayIT.java | 1 -
.../rocketmq/test/offset/OffsetNotFoundIT.java | 132 ++
.../test/smoke/NormalMessageSendAndRecvIT.java | 6 +
.../rocketmq/test/statictopic/StaticTopicIT.java | 521 ++++++
tools/pom.xml | 6 +-
.../rocketmq/tools/admin/DefaultMQAdminExt.java | 208 ++-
.../tools/admin/DefaultMQAdminExtImpl.java | 1149 ++++++++----
.../apache/rocketmq/tools/admin/MQAdminExt.java | 128 +-
.../apache/rocketmq/tools/admin/MQAdminUtils.java | 343 ++++
.../tools/admin/api/BrokerOperatorResult.java | 37 +-
.../tools/admin/common/AdminToolHandler.java | 10 +-
.../tools/admin/common/AdminToolResult.java | 76 +
.../common/AdminToolsResultCodeEnum.java} | 31 +-
.../apache/rocketmq/tools/command/CommandUtil.java | 17 +-
.../rocketmq/tools/command/MQAdminStartup.java | 24 +-
.../tools/command/SubCommandException.java | 4 +
.../command/broker/GetBrokerConfigCommand.java | 29 +-
.../broker/ResetMasterFlushOffsetSubCommand.java | 71 +
.../broker/UpdateBrokerConfigSubCommand.java | 16 +-
.../command/cluster/CLusterSendMsgRTCommand.java | 4 +-
.../command/cluster/ClusterListSubCommand.java | 106 +-
.../consumer/ConsumerProgressSubCommand.java | 17 +-
.../consumer/DeleteSubscriptionGroupCommand.java | 19 +-
.../consumer/GetConsumerConfigSubCommand.java | 2 +-
.../SetConsumeModeSubCommand.java} | 71 +-
.../command/consumer/UpdateSubGroupSubCommand.java | 26 +
.../command/container/AddBrokerSubCommand.java | 66 +
.../command/container/RemoveBrokerSubCommand.java | 79 +
.../tools/command/ha/HAStatusSubCommand.java | 151 ++
.../command/offset/ResetOffsetByTimeCommand.java | 34 +
.../tools/command/topic/DeleteTopicSubCommand.java | 2 +-
.../topic/RemappingStaticTopicSubCommand.java | 207 +++
.../command/topic/UpdateStaticTopicSubCommand.java | 208 +++
.../tools/command/topic/UpdateTopicSubCommand.java | 14 +
.../tools/admin/DefaultMQAdminExtTest.java | 74 +-
.../rocketmq/tools/command/CommandUtilTest.java | 2 +-
.../consumer/ConsumerProgressSubCommandTest.java | 2 +
.../message/QueryMsgByUniqueKeySubCommandTest.java | 18 +-
602 files changed, 52645 insertions(+), 9325 deletions(-)
diff --cc broker/pom.xml
index de2063114,8e52c1904..14a0b70f9
--- a/broker/pom.xml
+++ b/broker/pom.xml
@@@ -46,10 -46,10 +46,14 @@@
<groupId>${project.groupId}</groupId>
<artifactId>rocketmq-filter</artifactId>
</dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-acl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
@@@ -70,6 -74,14 +74,10 @@@
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.googlecode.concurrentlinkedhashmap</groupId>
+ <artifactId>concurrentlinkedhashmap-lru</artifactId>
+ </dependency>
- <dependency>
- <groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-acl</artifactId>
- </dependency>
</dependencies>
<build>
diff --cc broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
index c5b53a777,78e1dec27..9bfcc0f21
--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
@@@ -57,21 -93,36 +93,36 @@@ import org.apache.rocketmq.remoting.net
import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+ import java.io.UnsupportedEncodingException;
+ import java.util.Arrays;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.Set;
+ import java.util.concurrent.ArrayBlockingQueue;
+ import java.util.concurrent.CopyOnWriteArrayList;
+ import java.util.concurrent.CountDownLatch;
+ import java.util.concurrent.TimeUnit;
+
public class BrokerOuterAPI {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final RemotingClient remotingClient;
- private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr());
+ private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr());
private String nameSrvAddr = null;
private BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES,
- new ArrayBlockingQueue<Runnable>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true));
+ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true));
+ private ClientMetadata clientMetadata;
+ private RpcClient rpcClient;
+
public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
- this(nettyClientConfig, null);
+ this(nettyClientConfig, null, new ClientMetadata());
}
- public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook) {
+ private BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook, ClientMetadata clientMetadata) {
this.remotingClient = new NettyRemotingClient(nettyClientConfig);
+ this.clientMetadata = clientMetadata;
this.remotingClient.registerRPCHook(rpcHook);
+ this.rpcClient = new RpcClientImpl(this.clientMetadata, this.remotingClient);
}
public void start() {
@@@ -101,13 -156,202 +156,202 @@@
}
public void updateNameServerAddressList(final String addrs) {
- List<String> lst = new ArrayList<String>();
String[] addrArray = addrs.split(";");
- for (String addr : addrArray) {
- lst.add(addr);
+ List<String> lst = new ArrayList<String>(Arrays.asList(addrArray));
+ this.remotingClient.updateNameServerAddressList(lst);
+ }
+
+ public BrokerMemberGroup syncBrokerMemberGroup(String clusterName, String brokerName)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ return syncBrokerMemberGroup(clusterName, brokerName, false);
+ }
+
+ public BrokerMemberGroup syncBrokerMemberGroup(String clusterName, String brokerName,
+ boolean isCompatibleWithOldNameSrv)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ if (isCompatibleWithOldNameSrv) {
+ return getBrokerMemberGroupCompatible(clusterName, brokerName);
+ } else {
+ return getBrokerMemberGroup(clusterName, brokerName);
}
+ }
- this.remotingClient.updateNameServerAddressList(lst);
+ public BrokerMemberGroup getBrokerMemberGroup(String clusterName, String brokerName)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ BrokerMemberGroup brokerMemberGroup = new BrokerMemberGroup(clusterName, brokerName);
+
+ GetBrokerMemberGroupRequestHeader requestHeader = new GetBrokerMemberGroupRequestHeader();
+ requestHeader.setClusterName(clusterName);
+ requestHeader.setBrokerName(brokerName);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_MEMBER_GROUP, requestHeader);
+
+ RemotingCommand response = null;
+ response = this.remotingClient.invokeSync(null, request, 3000);
+ assert response != null;
+
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ byte[] body = response.getBody();
+ if (body != null) {
+ GetBrokerMemberGroupResponseBody brokerMemberGroupResponseBody =
+ GetBrokerMemberGroupResponseBody.decode(body, GetBrokerMemberGroupResponseBody.class);
+
+ return brokerMemberGroupResponseBody.getBrokerMemberGroup();
+ }
+ }
+ default:
+ break;
+ }
+
+ return brokerMemberGroup;
+ }
+
+ public BrokerMemberGroup getBrokerMemberGroupCompatible(String clusterName, String brokerName)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ BrokerMemberGroup brokerMemberGroup = new BrokerMemberGroup(clusterName, brokerName);
+
+ GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();
+ requestHeader.setTopic(TopicValidator.SYNC_BROKER_MEMBER_GROUP_PREFIX + brokerName);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);
+
+ RemotingCommand response;
+ response = this.remotingClient.invokeSync(null, request, 3000);
+ assert response != null;
+
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ byte[] body = response.getBody();
+ if (body != null) {
+ TopicRouteData topicRouteData = TopicRouteData.decode(body, TopicRouteData.class);
+ for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
+ if (brokerData != null
+ && brokerData.getBrokerName().equals(brokerName)
+ && brokerData.getCluster().equals(clusterName)) {
+ brokerMemberGroup.getBrokerAddrs().putAll(brokerData.getBrokerAddrs());
+ break;
+ }
+ }
+ return brokerMemberGroup;
+ }
+ }
+ default:
+ break;
+ }
+
+ return brokerMemberGroup;
+ }
+
+ public void sendHeartbeatViaDataVersion(
+ final String clusterName,
+ final String brokerAddr,
+ final String brokerName,
+ final Long brokerId,
+ final int timeoutMillis,
+ final DataVersion dataVersion,
+ final boolean isInBrokerContainer) {
+ List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
+ if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
+ final QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
+ requestHeader.setBrokerAddr(brokerAddr);
+ requestHeader.setBrokerName(brokerName);
+ requestHeader.setBrokerId(brokerId);
+ requestHeader.setClusterName(clusterName);
+
+ for (final String namesrvAddr : nameServerAddressList) {
+ brokerOuterExecutor.execute(new AbstractBrokerRunnable(new BrokerIdentity(clusterName, brokerName, brokerId, isInBrokerContainer)) {
+
+ @Override
+ public void run2() {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);
+ request.setBody(dataVersion.encode());
+
+ try {
+ BrokerOuterAPI.this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMillis);
+ } catch (Exception e) {
+ LOGGER.error("sendHeartbeat Exception " + namesrvAddr, e);
+ }
+ }
+ });
+ }
+ }
+ }
+
+ public void sendHeartbeat(final String clusterName,
+ final String brokerAddr,
+ final String brokerName,
+ final Long brokerId,
+ final int timeoutMills,
+ final boolean isInBrokerContainer) {
+ List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
+
+ final BrokerHeartbeatRequestHeader requestHeader = new BrokerHeartbeatRequestHeader();
+ requestHeader.setClusterName(clusterName);
+ requestHeader.setBrokerAddr(brokerAddr);
+ requestHeader.setBrokerName(brokerName);
+
+ if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
+ for (final String namesrvAddr : nameServerAddressList) {
+ brokerOuterExecutor.execute(new AbstractBrokerRunnable(new BrokerIdentity(clusterName, brokerName, brokerId, isInBrokerContainer)) {
+ @Override
+ public void run2() {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BROKER_HEARTBEAT, requestHeader);
+
+ try {
+ BrokerOuterAPI.this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMills);
+ } catch (Exception e) {
+ LOGGER.error("sendHeartbeat Exception " + namesrvAddr, e);
+ }
+ }
+ });
+ }
+ }
+ }
+
+ public BrokerSyncInfo retrieveBrokerHaInfo(String masterBrokerAddr)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,
+ MQBrokerException, RemotingCommandException {
+ ExchangeHAInfoRequestHeader requestHeader = new ExchangeHAInfoRequestHeader();
+ requestHeader.setMasterHaAddress(null);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(masterBrokerAddr, request, 3000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
- ExchangeHAInfoResponseHeader responseHeader = response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
++ ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
+ return new BrokerSyncInfo(responseHeader.getMasterHaAddress(), responseHeader.getMasterFlushOffset(), responseHeader.getMasterAddress());
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public void sendBrokerHaInfo(String brokerAddr, String masterHaAddr, long brokerInitMaxOffset, String masterAddr)
+ throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
+ ExchangeHAInfoRequestHeader requestHeader = new ExchangeHAInfoRequestHeader();
+ requestHeader.setMasterHaAddress(masterHaAddr);
+ requestHeader.setMasterFlushOffset(brokerInitMaxOffset);
+ requestHeader.setMasterAddress(masterAddr);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000);
+
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ return;
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
}
public List<RegisterBrokerResult> registerBrokerAll(
@@@ -388,4 -689,303 +689,303 @@@
public void registerRPCHook(RPCHook rpcHook) {
remotingClient.registerRPCHook(rpcHook);
}
+
+ public void clearRPCHook() {
+ remotingClient.clearRPCHook();
+ }
+
+ public long getMaxOffset(final String addr, final String topic, final int queueId, final boolean committed,
+ final boolean isOnlyThisBroker)
+ throws RemotingException, MQBrokerException, InterruptedException {
+ GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();
+ requestHeader.setTopic(topic);
+ requestHeader.setQueueId(queueId);
+ requestHeader.setCommitted(committed);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
- GetMaxOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
++ GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
+
+ return responseHeader.getOffset();
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public long getMinOffset(final String addr, final String topic, final int queueId, final boolean isOnlyThisBroker)
+ throws RemotingException, MQBrokerException, InterruptedException {
+ GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader();
+ requestHeader.setTopic(topic);
+ requestHeader.setQueueId(queueId);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
- GetMinOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
++ GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
+
+ return responseHeader.getOffset();
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public void lockBatchMQAsync(
+ final String addr,
+ final LockBatchRequestBody requestBody,
+ final long timeoutMillis,
+ final LockCallback callback) throws RemotingException, InterruptedException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);
+
+ request.setBody(requestBody.encode());
+ this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
+ if (callback == null) {
+ return;
+ }
+
+ try {
+ RemotingCommand response = responseFuture.getResponseCommand();
+ if (response != null) {
+ if (response.getCode() == ResponseCode.SUCCESS) {
+ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(),
+ LockBatchResponseBody.class);
+ Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
+ callback.onSuccess(messageQueues);
+ } else {
+ callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
+ }
+ }
+ } catch (Throwable ignored) {
+
+ }
+ });
+ }
+
+ public void unlockBatchMQAsync(
+ final String addr,
+ final UnlockBatchRequestBody requestBody,
+ final long timeoutMillis,
+ final UnlockCallback callback) throws RemotingException, InterruptedException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, null);
+
+ request.setBody(requestBody.encode());
+
+ this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
+ if (callback == null) {
+ return;
+ }
+
+ try {
+ RemotingCommand response = responseFuture.getResponseCommand();
+ if (response != null) {
+ if (response.getCode() == ResponseCode.SUCCESS) {
+ callback.onSuccess();
+ } else {
+ callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
+ }
+ }
+ } catch (Throwable ignored) {
+
+ }
+ });
+ }
+
+ public RemotingClient getRemotingClient() {
+ return this.remotingClient;
+ }
+
+ public SendResult sendMessageToSpecificBroker(String brokerAddr, final String brokerName,
+ final MessageExt msg, String group,
+ long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException {
+
+ SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+ requestHeader.setProducerGroup(group);
+ requestHeader.setTopic(msg.getTopic());
+ requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);
+ requestHeader.setDefaultTopicQueueNums(8);
+ requestHeader.setQueueId(msg.getQueueId());
+ requestHeader.setSysFlag(msg.getSysFlag());
+ requestHeader.setBornTimestamp(msg.getBornTimestamp());
+ requestHeader.setFlag(msg.getFlag());
+ requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));
+ requestHeader.setReconsumeTimes(msg.getReconsumeTimes());
+ requestHeader.setBatch(false);
+
+ SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader);
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2);
+
+ request.setBody(msg.getBody());
+
+ RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
+
+ return this.processSendResponse(brokerName, msg, response);
+ }
+
+ private SendResult processSendResponse(
+ final String brokerName,
+ final Message msg,
+ final RemotingCommand response
+ ) throws MQBrokerException, RemotingCommandException {
+ switch (response.getCode()) {
+ case ResponseCode.FLUSH_DISK_TIMEOUT:
+ case ResponseCode.FLUSH_SLAVE_TIMEOUT:
+ case ResponseCode.SLAVE_NOT_AVAILABLE:
+ case ResponseCode.SUCCESS: {
+ SendStatus sendStatus = SendStatus.SEND_OK;
+ switch (response.getCode()) {
+ case ResponseCode.FLUSH_DISK_TIMEOUT:
+ sendStatus = SendStatus.FLUSH_DISK_TIMEOUT;
+ break;
+ case ResponseCode.FLUSH_SLAVE_TIMEOUT:
+ sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT;
+ break;
+ case ResponseCode.SLAVE_NOT_AVAILABLE:
+ sendStatus = SendStatus.SLAVE_NOT_AVAILABLE;
+ break;
+ case ResponseCode.SUCCESS:
+ sendStatus = SendStatus.SEND_OK;
+ break;
+ default:
+ assert false;
+ break;
+ }
+
+ SendMessageResponseHeader responseHeader =
+ (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
+
+ //If namespace not null , reset Topic without namespace.
+ String topic = msg.getTopic();
+
+ MessageQueue messageQueue = new MessageQueue(topic, brokerName, responseHeader.getQueueId());
+
+ String uniqMsgId = MessageClientIDSetter.getUniqID(msg);
+ if (msg instanceof MessageBatch) {
+ StringBuilder sb = new StringBuilder();
+ for (Message message : (MessageBatch) msg) {
+ sb.append(sb.length() == 0 ? "" : ",").append(MessageClientIDSetter.getUniqID(message));
+ }
+ uniqMsgId = sb.toString();
+ }
+ SendResult sendResult = new SendResult(sendStatus,
+ uniqMsgId,
+ responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());
+ sendResult.setTransactionId(responseHeader.getTransactionId());
+ String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);
+ String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH);
+ if (regionId == null || regionId.isEmpty()) {
+ regionId = MixAll.DEFAULT_TRACE_REGION_ID;
+ }
+ if (traceOn != null && traceOn.equals("false")) {
+ sendResult.setTraceOn(false);
+ } else {
+ sendResult.setTraceOn(true);
+ }
+ sendResult.setRegionId(regionId);
+ return sendResult;
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public BrokerFixedThreadPoolExecutor getBrokerOuterExecutor() {
+ return brokerOuterExecutor;
+ }
+
+ public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis)
+ throws RemotingException, MQBrokerException, InterruptedException {
+ return getTopicRouteInfoFromNameServer(topic, timeoutMillis, true);
+ }
+
+ public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis,
+ boolean allowTopicNotExist) throws MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+ GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();
+ requestHeader.setTopic(topic);
+
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);
+
+ RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.TOPIC_NOT_EXIST: {
+ if (allowTopicNotExist) {
+ LOGGER.warn("get Topic [{}] RouteInfoFromNameServer is not exist value", topic);
+ }
+
+ break;
+ }
+ case ResponseCode.SUCCESS: {
+ byte[] body = response.getBody();
+ if (body != null) {
+ return TopicRouteData.decode(body, TopicRouteData.class);
+ }
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public ClusterInfo getBrokerClusterInfo() throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null);
+ RemotingCommand response = this.remotingClient.invokeSync(null, request, 3_000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ return ClusterInfo.decode(response.getBody(), ClusterInfo.class);
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark());
+ }
+
+ public void forwardRequest(String brokerAddr, RemotingCommand request, long timeoutMillis,
+ InvokeCallback invokeCallback) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException, RemotingTooMuchRequestException, RemotingConnectException {
+ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, invokeCallback);
+ }
+
+ public void refreshMetadata() throws Exception {
+ ClusterInfo brokerClusterInfo = getBrokerClusterInfo();
+ clientMetadata.refreshClusterInfo(brokerClusterInfo);
+ }
+
+ public ClientMetadata getClientMetadata() {
+ return clientMetadata;
+ }
+
+ public RpcClient getRpcClient() {
+ return rpcClient;
+ }
+
+ public MessageRequestModeSerializeWrapper getAllMessageRequestMode(
+ final String addr) throws RemotingSendRequestException, RemotingConnectException,
+ MQBrokerException, RemotingTimeoutException, InterruptedException {
+ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_MESSAGE_REQUEST_MODE, null);
+ RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+ assert response != null;
+ switch (response.getCode()) {
+ case ResponseCode.SUCCESS: {
+ return MessageRequestModeSerializeWrapper.decode(response.getBody(), MessageRequestModeSerializeWrapper.class);
+ }
+ default:
+ break;
+ }
+
+ throw new MQBrokerException(response.getCode(), response.getRemark(), addr);
+ }
}
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
index 21832e27a,18212397d..3d1267818
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
@@@ -220,10 -512,10 +512,10 @@@ public abstract class AbstractSendMessa
if (queueIdInt >= idValid) {
String errorInfo = String.format("request queueId[%d] is illegal, %s Producer: %s",
queueIdInt,
- topicConfig.toString(),
+ topicConfig,
RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
- log.warn(errorInfo);
+ LOGGER.warn(errorInfo);
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(errorInfo);
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
index ca99605f7,0917858f7..ed94e6771
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
@@@ -26,11 -31,11 +31,14 @@@ import org.apache.rocketmq.broker.clien
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.broker.filter.ConsumerFilterData;
import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
+ import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;
+ import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
+import org.apache.rocketmq.common.protocol.header.GetAllProducerInfoRequestHeader;
+import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;
import org.apache.rocketmq.common.AclConfig;
+ import org.apache.rocketmq.common.LockCallback;
import org.apache.rocketmq.common.MQVersion;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.PlainAccessConfig;
@@@ -104,9 -126,20 +129,19 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader;
import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+ import org.apache.rocketmq.common.rpc.RpcClientUtils;
+ import org.apache.rocketmq.common.rpc.RpcException;
+ import org.apache.rocketmq.common.rpc.RpcRequest;
+ import org.apache.rocketmq.common.rpc.RpcResponse;
+ import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
+ import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
import org.apache.rocketmq.common.stats.StatsItem;
import org.apache.rocketmq.common.stats.StatsSnapshot;
+ import org.apache.rocketmq.common.subscription.GroupForbidden;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
-import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.filter.util.BitsArray;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
@@@ -138,12 -172,14 +174,15 @@@ import java.util.List
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentMap;
+import org.apache.rocketmq.store.config.BrokerRole;
- public class AdminBrokerProcessor extends AsyncNettyRequestProcessor {
- private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
- private final BrokerController brokerController;
+ import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
+
+ public class AdminBrokerProcessor implements NettyRequestProcessor {
+ private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ protected final BrokerController brokerController;
public AdminBrokerProcessor(final BrokerController brokerController) {
this.brokerController = brokerController;
@@@ -254,12 -372,11 +379,14 @@@
private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ if (validateSlave(response)) {
+ return response;
+ }
final CreateTopicRequestHeader requestHeader =
(CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class);
- log.info("updateAndCreateTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ LOGGER.info("Broker receive request to update or create topic={}, caller address={}",
+ requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
String topic = requestHeader.getTopic();
@@@ -691,11 -1164,9 +1178,12 @@@
private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request)
throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ if (validateSlave(response)) {
+ return response;
+ }
- log.info("updateAndCreateSubscriptionGroup called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+ LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}",
+ RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
SubscriptionGroupConfig config = RemotingSerializable.decode(request.getBody(), SubscriptionGroupConfig.class);
if (config != null) {
@@@ -1188,21 -1735,11 +1772,21 @@@
return response;
}
+ public RemotingCommand deleteExpiredCommitLog() {
- log.warn("invoke deleteExpiredCommitLog start.");
++ LOGGER.warn("invoke deleteExpiredCommitLog start.");
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ brokerController.getMessageStore().executeDeleteFilesManually();
- log.warn("invoke deleteExpiredCommitLog end.");
++ LOGGER.warn("invoke deleteExpiredCommitLog end.");
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
+
public RemotingCommand cleanUnusedTopic() {
- log.warn("invoke cleanUnusedTopic start.");
+ LOGGER.warn("invoke cleanUnusedTopic start.");
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
brokerController.getMessageStore().cleanUnusedTopic(brokerController.getTopicConfigManager().getTopicConfigTable().keySet());
- log.warn("invoke cleanUnusedTopic end.");
+ LOGGER.warn("invoke cleanUnusedTopic end.");
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
return response;
@@@ -1683,13 -2259,116 +2306,126 @@@
return inner;
}
+ private RemotingCommand getTopicConfig(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ GetTopicConfigRequestHeader requestHeader = (GetTopicConfigRequestHeader) request.decodeCommandCustomHeader(GetTopicConfigRequestHeader.class);
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(requestHeader.getTopic());
+ if (topicConfig == null) {
+ LOGGER.error("No topic in this broker, client: {} topic: {}", ctx.channel().remoteAddress(), requestHeader.getTopic());
+ //be care of the response code, should set "not-exist" explictly
+ response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+ response.setRemark("No topic in this broker. topic: " + requestHeader.getTopic());
+ return response;
+ }
+ TopicQueueMappingDetail topicQueueMappingDetail = null;
+ if (Boolean.TRUE.equals(requestHeader.getLo())) {
+ topicQueueMappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(requestHeader.getTopic());
+ }
+ String content = JSONObject.toJSONString(new TopicConfigAndQueueMapping(topicConfig, topicQueueMappingDetail));
+ try {
+ response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.error("UnsupportedEncodingException getTopicConfig: topic=" + topicConfig.getTopicName(), e);
+
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("UnsupportedEncodingException " + e.getMessage());
+ return response;
+ }
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+
+ return response;
+ }
+
+ private RemotingCommand notifyMinBrokerIdChange(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
- NotifyMinBrokerIdChangeRequestHeader requestHeader = request.decodeCommandCustomHeader(NotifyMinBrokerIdChangeRequestHeader.class);
++ NotifyMinBrokerIdChangeRequestHeader requestHeader = (NotifyMinBrokerIdChangeRequestHeader) request.decodeCommandCustomHeader(NotifyMinBrokerIdChangeRequestHeader.class);
+
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ LOGGER.warn("min broker id changed, prev {}, new {}", this.brokerController.getMinBrokerIdInGroup(), requestHeader.getMinBrokerId());
+
+ this.brokerController.updateMinBroker(requestHeader.getMinBrokerId(), requestHeader.getMinBrokerAddr(),
+ requestHeader.getOfflineBrokerAddr(),
+ requestHeader.getHaBrokerAddr());
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+
+ return response;
+ }
+
+ private RemotingCommand updateBrokerHaInfo(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ RemotingCommand response = RemotingCommand.createResponseCommand(ExchangeHAInfoResponseHeader.class);
+
- ExchangeHAInfoRequestHeader requestHeader = request.decodeCommandCustomHeader(ExchangeHAInfoRequestHeader.class);
++ ExchangeHAInfoRequestHeader requestHeader = (ExchangeHAInfoRequestHeader) request.decodeCommandCustomHeader(ExchangeHAInfoRequestHeader.class);
+ if (requestHeader.getMasterHaAddress() != null) {
+ this.brokerController.getMessageStore().updateHaMasterAddress(requestHeader.getMasterHaAddress());
+ this.brokerController.getMessageStore().updateMasterAddress(requestHeader.getMasterAddress());
+ if (this.brokerController.getMessageStore().getMasterFlushedOffset() == 0
+ && this.brokerController.getMessageStoreConfig().isSyncMasterFlushOffsetWhenStartup()) {
+ LOGGER.info("Set master flush offset in slave to {}", requestHeader.getMasterFlushOffset());
+ this.brokerController.getMessageStore().setMasterFlushedOffset(requestHeader.getMasterFlushOffset());
+ }
+ } else if (this.brokerController.getBrokerConfig().getBrokerId() == MixAll.MASTER_ID) {
+ final ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.readCustomHeader();
+ responseHeader.setMasterHaAddress(this.brokerController.getHAServerAddr());
+ responseHeader.setMasterFlushOffset(this.brokerController.getMessageStore().getBrokerInitMaxOffset());
+ responseHeader.setMasterAddress(this.brokerController.getBrokerAddr());
+ }
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+
+ return response;
+ }
+
+ private RemotingCommand getBrokerHaStatus(ChannelHandlerContext ctx, RemotingCommand request) {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ HARuntimeInfo runtimeInfo = this.brokerController.getMessageStore().getHARuntimeInfo();
+
+ if (runtimeInfo != null) {
+ byte[] body = runtimeInfo.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("Can not get HARuntimeInfo, may be duplicationEnable is true");
+ }
+
+ return response;
+ }
+
+ private RemotingCommand resetMasterFlushOffset(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ if (this.brokerController.getBrokerConfig().getBrokerId() != MixAll.MASTER_ID) {
+
- ResetMasterFlushOffsetHeader requestHeader = request.decodeCommandCustomHeader(ResetMasterFlushOffsetHeader.class);
++ ResetMasterFlushOffsetHeader requestHeader = (ResetMasterFlushOffsetHeader) request.decodeCommandCustomHeader(ResetMasterFlushOffsetHeader.class);
+
+ if (requestHeader.getMasterFlushOffset() != null) {
+ this.brokerController.getMessageStore().setMasterFlushedOffset(requestHeader.getMasterFlushOffset());
+ }
+ }
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
++
+ private boolean validateSlave(RemotingCommand response) {
+ if (this.brokerController.getMessageStoreConfig().getBrokerRole().equals(BrokerRole.SLAVE)) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("Can't modify topic or subscription group from slave broker, " +
+ "please execute it from master broker.");
+ return true;
+ }
+ return false;
+ }
}
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
index f0836f86e,7033417b5..0e8bac793
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
@@@ -75,6 -79,200 +79,200 @@@ public class PullMessageProcessor imple
public PullMessageProcessor(final BrokerController brokerController) {
this.brokerController = brokerController;
+ this.pullMessageResultHandler = new DefaultPullMessageResultHandler(brokerController);
+ }
+
+ private RemotingCommand rewriteRequestForStaticTopic(PullMessageRequestHeader requestHeader,
+ TopicQueueMappingContext mappingContext) {
+ try {
+ if (mappingContext.getMappingDetail() == null) {
+ return null;
+ }
+ TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+ String topic = mappingContext.getTopic();
+ Integer globalId = mappingContext.getGlobalId();
+ // if the leader? consider the order consumer, which will lock the mq
+ if (!mappingContext.isLeader()) {
+ return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d cannot find mapping item in request process of current broker %s", topic, globalId, mappingDetail.getBname()));
+ }
- Long globalOffset = requestHeader.getQueueOffset();
+
++ Long globalOffset = requestHeader.getQueueOffset();
+ LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), globalOffset, true);
+ mappingContext.setCurrentItem(mappingItem);
+
+ if (globalOffset < mappingItem.getLogicOffset()) {
+ //handleOffsetMoved
+ //If the physical queue is reused, we should handle the PULL_OFFSET_MOVED independently
+ //Otherwise, we could just transfer it to the physical process
+ }
+ //below are physical info
+ String bname = mappingItem.getBname();
+ Integer phyQueueId = mappingItem.getQueueId();
+ Long phyQueueOffset = mappingItem.computePhysicalQueueOffset(globalOffset);
+ requestHeader.setQueueId(phyQueueId);
+ requestHeader.setQueueOffset(phyQueueOffset);
+ if (mappingItem.checkIfEndOffsetDecided()
+ && requestHeader.getMaxMsgNums() != null) {
+ requestHeader.setMaxMsgNums((int) Math.min(mappingItem.getEndOffset() - mappingItem.getStartOffset(), requestHeader.getMaxMsgNums()));
+ }
+
+ if (mappingDetail.getBname().equals(bname)) {
+ //just let it go, do the local pull process
+ return null;
+ }
+
+ int sysFlag = requestHeader.getSysFlag();
+ requestHeader.setLo(false);
+ requestHeader.setBname(bname);
+ sysFlag = PullSysFlag.clearSuspendFlag(sysFlag);
+ sysFlag = PullSysFlag.clearCommitOffsetFlag(sysFlag);
+ requestHeader.setSysFlag(sysFlag);
+ RpcRequest rpcRequest = new RpcRequest(RequestCode.PULL_MESSAGE, requestHeader, null);
+ RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();
+ if (rpcResponse.getException() != null) {
+ throw rpcResponse.getException();
+ }
+
+ PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) rpcResponse.getHeader();
+ {
+ RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, rpcResponse.getCode());
+ if (rewriteResult != null) {
+ return rewriteResult;
+ }
+ }
+ return RpcClientUtils.createCommandForRpcResponse(rpcResponse);
+ } catch (Throwable t) {
+ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+ }
+ }
+
+ private RemotingCommand rewriteResponseForStaticTopic(PullMessageRequestHeader requestHeader,
+ PullMessageResponseHeader responseHeader,
+ TopicQueueMappingContext mappingContext, final int code) {
+ try {
+ if (mappingContext.getMappingDetail() == null) {
+ return null;
+ }
+ TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+ LogicQueueMappingItem leaderItem = mappingContext.getLeaderItem();
+
+ LogicQueueMappingItem currentItem = mappingContext.getCurrentItem();
+
+ LogicQueueMappingItem earlistItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true);
+
+ assert currentItem.getLogicOffset() >= 0;
+
+ long requestOffset = requestHeader.getQueueOffset();
+ long nextBeginOffset = responseHeader.getNextBeginOffset();
+ long minOffset = responseHeader.getMinOffset();
+ long maxOffset = responseHeader.getMaxOffset();
+ int responseCode = code;
+
+ //consider the following situations
+ // 1. read from slave, currently not supported
+ // 2. the middle queue is truncated because of deleting commitlog
+ if (code != ResponseCode.SUCCESS) {
+ //note the currentItem maybe both the leader and the earliest
+ boolean isRevised = false;
+ if (leaderItem.getGen() == currentItem.getGen()) {
+ //read the leader
+ if (requestOffset > maxOffset) {
+ //actually, we need do nothing, but keep the code structure here
+ if (code == ResponseCode.PULL_OFFSET_MOVED) {
+ responseCode = ResponseCode.PULL_OFFSET_MOVED;
+ nextBeginOffset = maxOffset;
+ } else {
+ //maybe current broker is the slave
+ responseCode = code;
+ }
+ } else if (requestOffset < minOffset) {
+ nextBeginOffset = minOffset;
+ responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+ } else {
+ responseCode = code;
+ }
+ }
+ //note the currentItem maybe both the leader and the earliest
+ if (earlistItem.getGen() == currentItem.getGen()) {
+ //read the earliest one
+ if (requestOffset < minOffset) {
+ if (code == ResponseCode.PULL_OFFSET_MOVED) {
+ responseCode = ResponseCode.PULL_OFFSET_MOVED;
+ nextBeginOffset = minOffset;
+ } else {
+ //maybe read from slave, but we still set it to moved
+ responseCode = ResponseCode.PULL_OFFSET_MOVED;
+ nextBeginOffset = minOffset;
+ }
+ } else if (requestOffset >= maxOffset) {
+ //just move to another item
+ LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);
+ if (nextItem != null) {
+ isRevised = true;
+ currentItem = nextItem;
+ nextBeginOffset = currentItem.getStartOffset();
+ minOffset = currentItem.getStartOffset();
+ maxOffset = minOffset;
+ responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+ } else {
+ //maybe the next one's logic offset is -1
+ responseCode = ResponseCode.PULL_NOT_FOUND;
+ }
+ } else {
+ //let it go
+ responseCode = code;
+ }
+ }
+
+ //read from the middle item, ignore the PULL_OFFSET_MOVED
+ if (!isRevised
+ && leaderItem.getGen() != currentItem.getGen()
+ && earlistItem.getGen() != currentItem.getGen()) {
+ if (requestOffset < minOffset) {
+ nextBeginOffset = minOffset;
+ responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+ } else if (requestOffset >= maxOffset) {
+ //just move to another item
+ LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);
+ if (nextItem != null) {
+ currentItem = nextItem;
+ nextBeginOffset = currentItem.getStartOffset();
+ minOffset = currentItem.getStartOffset();
+ maxOffset = minOffset;
+ responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+ } else {
+ //maybe the next one's logic offset is -1
+ responseCode = ResponseCode.PULL_NOT_FOUND;
+ }
+ } else {
+ responseCode = code;
+ }
+ }
+ }
+
+ //handle nextBeginOffset
+ //the next begin offset should no more than the end offset
+ if (currentItem.checkIfEndOffsetDecided()
+ && nextBeginOffset >= currentItem.getEndOffset()) {
+ nextBeginOffset = currentItem.getEndOffset();
+ }
+ responseHeader.setNextBeginOffset(currentItem.computeStaticQueueOffsetStrictly(nextBeginOffset));
+ //handle min offset
+ responseHeader.setMinOffset(currentItem.computeStaticQueueOffsetStrictly(Math.max(currentItem.getStartOffset(), minOffset)));
+ //handle max offset
+ responseHeader.setMaxOffset(Math.max(currentItem.computeStaticQueueOffsetStrictly(maxOffset),
+ TopicQueueMappingDetail.computeMaxOffsetFromMapping(mappingDetail, mappingContext.getGlobalId())));
+ //set the offsetDelta
+ responseHeader.setOffsetDelta(currentItem.computeOffsetDelta());
+
+ if (code != ResponseCode.SUCCESS) {
+ return RemotingCommand.createResponseCommandWithHeader(responseCode, responseHeader);
+ } else {
+ return null;
+ }
+ } catch (Throwable t) {
+ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+ }
}
@Override
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
index 133165b9f,a088973fb..183f64f38
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
@@@ -39,8 -39,9 +39,8 @@@ import org.apache.rocketmq.logging.Inte
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.exception.RemotingException;
-import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
- import org.apache.rocketmq.store.MessageExtBrokerInner;
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
diff --cc broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
index 2e009972d,437990c93..e94759500
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
@@@ -440,11 -252,13 +252,13 @@@ public class SendMessageProcessor exten
msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName);
+
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
- PutMessageResult putMessageResult = null;
- Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
+
+ // Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
+ boolean sendTransactionPrepareMessage = false;
- if (traFlag != null && Boolean.parseBoolean(traFlag)
+ if (Boolean.parseBoolean(traFlag)
&& !(msgInner.getReconsumeTimes() > 0 && msgInner.getDelayTimeLevel() > 0)) { //For client under version 4.6.1
if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {
response.setCode(ResponseCode.NO_PERMISSION);
diff --cc broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
index 35df22719,48709b3f4..e0436f1a7
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
@@@ -54,18 -60,11 +60,12 @@@ import org.apache.rocketmq.remoting.exc
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
- import org.apache.rocketmq.store.AppendMessageResult;
- import org.apache.rocketmq.store.AppendMessageStatus;
import org.apache.rocketmq.store.DefaultMessageStore;
- import org.apache.rocketmq.store.MappedFile;
- import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.MessageStore;
- import org.apache.rocketmq.store.PutMessageResult;
- import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.config.MessageStoreConfig;
- import org.apache.rocketmq.store.schedule.ScheduleMessageService;
+ import org.apache.rocketmq.store.logfile.DefaultMappedFile;
import org.apache.rocketmq.store.stats.BrokerStats;
import org.junit.Before;
import org.junit.Test;
diff --cc broker/src/test/java/org/apache/rocketmq/broker/schedule/ScheduleMessageServiceTest.java
index e0f4a492e,c743fca25..cff1374b7
--- a/broker/src/test/java/org/apache/rocketmq/broker/schedule/ScheduleMessageServiceTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/schedule/ScheduleMessageServiceTest.java
@@@ -37,14 -27,37 +27,37 @@@ import java.nio.ByteBuffer
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+ import java.util.Random;
import java.util.UUID;
+ import java.util.concurrent.ConcurrentHashMap;
+ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
+ import org.apache.rocketmq.broker.BrokerController;
+ import org.apache.rocketmq.broker.failover.EscapeBridge;
+ import org.apache.rocketmq.broker.util.HookUtils;
+ import org.apache.rocketmq.common.BrokerConfig;
+ import org.apache.rocketmq.common.UtilAll;
+ import org.apache.rocketmq.common.message.MessageDecoder;
+ import org.apache.rocketmq.common.message.MessageExt;
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+ import org.apache.rocketmq.store.ConsumeQueueExt;
+ import org.apache.rocketmq.store.DefaultMessageStore;
+ import org.apache.rocketmq.store.GetMessageResult;
+ import org.apache.rocketmq.store.GetMessageStatus;
+ import org.apache.rocketmq.store.MessageArrivingListener;
+ import org.apache.rocketmq.store.PutMessageResult;
+ import org.apache.rocketmq.store.config.MessageStoreConfig;
+ import org.apache.rocketmq.store.stats.BrokerStatsManager;
+ import org.junit.After;
+ import org.junit.Before;
+ import org.junit.Test;
+ import org.mockito.Mockito;
-import static org.apache.rocketmq.store.stats.BrokerStatsManager.BROKER_PUT_NUMS;
-import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_NUMS;
-import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_SIZE;
+import static org.apache.rocketmq.common.stats.Stats.BROKER_PUT_NUMS;
+import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_NUMS;
+import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_SIZE;
import static org.assertj.core.api.Assertions.assertThat;
-
+ import static org.junit.Assert.*;
public class ScheduleMessageServiceTest {
diff --cc client/pom.xml
index 2b3b2be93,1e3d23e98..d695183f5
--- a/client/pom.xml
+++ b/client/pom.xml
@@@ -54,6 -51,12 +49,10 @@@
<dependency>
<groupId>io.opentracing</groupId>
<artifactId>opentracing-mock</artifactId>
- <version>0.33.0</version>
- <scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
</project>
diff --cc client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AbstractAllocateMessageQueueStrategy.java
index 22ba5060c,bd03e1ff5..e0a050091
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AbstractAllocateMessageQueueStrategy.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AbstractAllocateMessageQueueStrategy.java
@@@ -25,19 -23,30 +25,19 @@@ import org.apache.rocketmq.client.log.C
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.logging.InternalLogger;
-/**
- * Cycle average Hashing queue algorithm
- */
-public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy {
- private InternalLogger log;
-
- public AllocateMessageQueueAveragelyByCircle() {
- log = ClientLogger.getLog();
- }
+public abstract class AbstractAllocateMessageQueueStrategy implements AllocateMessageQueueStrategy {
- private final InternalLogger log = ClientLogger.getLog();
- public AllocateMessageQueueAveragelyByCircle(InternalLogger log) {
- this.log = log;
- }
++ protected InternalLogger log = ClientLogger.getLog();
- @Override
- public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
+ public boolean check(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
- if (currentCID == null || currentCID.length() < 1) {
+ if (StringUtils.isEmpty(currentCID)) {
throw new IllegalArgumentException("currentCID is empty");
}
- if (mqAll == null || mqAll.isEmpty()) {
+ if (CollectionUtils.isEmpty(mqAll)) {
throw new IllegalArgumentException("mqAll is null or mqAll empty");
}
- if (cidAll == null || cidAll.isEmpty()) {
+ if (CollectionUtils.isEmpty(cidAll)) {
throw new IllegalArgumentException("cidAll is null or cidAll empty");
}
diff --cc client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
index 895f27757,4c31041e7..d8abd0a01
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
@@@ -18,13 -18,25 +18,23 @@@ package org.apache.rocketmq.client.cons
import java.util.ArrayList;
import java.util.List;
-import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+ import org.apache.rocketmq.client.log.ClientLogger;
-import org.apache.rocketmq.common.message.MessageQueue;
+ import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.common.message.MessageQueue;
/**
* Average Hashing queue algorithm
*/
-public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy {
- private InternalLogger log;
+public class AllocateMessageQueueAveragely extends AbstractAllocateMessageQueueStrategy {
+ public AllocateMessageQueueAveragely() {
+ log = ClientLogger.getLog();
+ }
+
+ public AllocateMessageQueueAveragely(InternalLogger log) {
+ this.log = log;
+ }
+
@Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
diff --cc client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
index d23350074,bd03e1ff5..c07ed6809
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
@@@ -18,13 -18,25 +18,23 @@@ package org.apache.rocketmq.client.cons
import java.util.ArrayList;
import java.util.List;
-import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+ import org.apache.rocketmq.client.log.ClientLogger;
-import org.apache.rocketmq.common.message.MessageQueue;
+ import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.common.message.MessageQueue;
/**
* Cycle average Hashing queue algorithm
*/
-public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy {
- private InternalLogger log;
+public class AllocateMessageQueueAveragelyByCircle extends AbstractAllocateMessageQueueStrategy {
+ public AllocateMessageQueueAveragelyByCircle() {
+ log = ClientLogger.getLog();
+ }
+
+ public AllocateMessageQueueAveragelyByCircle(InternalLogger log) {
+ this.log = log;
+ }
+
@Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
diff --cc client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
index 5d7e1685e,15b32d850..4739225cd
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
@@@ -79,7 -92,8 +92,9 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+ import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody;
+ import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueryConsumeTimeSpanBody;
import org.apache.rocketmq.common.protocol.body.QueryCorrectionOffsetBody;
@@@ -98,7 -118,7 +119,8 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader;
import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader;
import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.common.protocol.header.GetAllProducerInfoRequestHeader;
import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader;
import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseBody;
import org.apache.rocketmq.common.protocol.header.GetConsumeStatsInBrokerHeader;
@@@ -151,11 -180,16 +182,17 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader;
import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
+ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+ import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+ import org.apache.rocketmq.common.subscription.GroupForbidden;
+import org.apache.rocketmq.common.rpchook.StreamTypeRPCHook;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+ import org.apache.rocketmq.common.sysflag.PullSysFlag;
import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.InvokeCallback;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.RemotingClient;
@@@ -396,9 -469,9 +476,9 @@@ public class MQClientAPIImpl implement
clusterAclVersionInfo.setBrokerAddr(responseHeader.getBrokerAddr());
clusterAclVersionInfo.setAclConfigDataVersion(DataVersion.fromJson(responseHeader.getVersion(), DataVersion.class));
HashMap<String, Object> dataVersionMap = JSON.parseObject(responseHeader.getAllAclFileVersion(), HashMap.class);
- Map<String, DataVersion> allAclConfigDataVersion = new HashMap<String, DataVersion>();
+ Map<String, DataVersion> allAclConfigDataVersion = new HashMap<String, DataVersion>(dataVersionMap.size(), 1);
for (Map.Entry<String, Object> entry : dataVersionMap.entrySet()) {
- allAclConfigDataVersion.put(entry.getKey(),DataVersion.fromJson(JSON.toJSONString(entry.getValue()), DataVersion.class));
+ allAclConfigDataVersion.put(entry.getKey(), DataVersion.fromJson(JSON.toJSONString(entry.getValue()), DataVersion.class));
}
clusterAclVersionInfo.setAllAclConfigDataVersion(allAclConfigDataVersion);
return clusterAclVersionInfo;
@@@ -629,10 -710,22 +717,10 @@@
}
String addr = instance.findBrokerAddressInPublish(retryBrokerName);
log.warn("async send msg by retry {} times. topic={}, brokerAddr={}, brokerName={}", tmp, msg.getTopic(), addr,
- retryBrokerName, e);
+ retryBrokerName, e);
- try {
- request.setOpaque(RemotingCommand.createNewRequestId());
- sendMessageAsync(addr, retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance,
- timesTotal, curTimes, context, producer);
- } catch (InterruptedException e1) {
- onExceptionImpl(retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1,
- context, false, producer);
- } catch (RemotingTooMuchRequestException e1) {
- onExceptionImpl(retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1,
- context, false, producer);
- } catch (RemotingException e1) {
- producer.updateFaultItem(brokerName, 3000, true);
- onExceptionImpl(retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1,
- context, true, producer);
- }
+ request.setOpaque(RemotingCommand.createNewRequestId());
+ sendMessageAsync(addr, retryBrokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance,
- timesTotal, curTimes, context, producer);
++ timesTotal, curTimes, context, producer);
} else {
if (context != null) {
diff --cc client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
index c3626c38b,4aca8f337..18bd8b3e4
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
@@@ -311,10 -317,10 +317,10 @@@ public class ConsumeMessageConcurrently
// Wrap topic with namespace before sending back message.
msg.setTopic(this.defaultMQPushConsumer.withNamespace(msg.getTopic()));
try {
- this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, context.getMessageQueue().getBrokerName());
+ this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, this.defaultMQPushConsumer.queueWithNamespace(context.getMessageQueue()));
return true;
} catch (Exception e) {
- log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg.toString(), e);
+ log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg, e);
}
return false;
diff --cc client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
index 668f9b6b6,325ca4fa6..3e23485cc
--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
@@@ -879,14 -887,9 +883,14 @@@ public class DefaultMQProducerImpl impl
}
}
- throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
+ throw new MQClientException("The broker[" + brokerName + "] not exist", null);
}
+ public MQClientInstance getMqClientFactory() {
+ return mQClientFactory;
+ }
+
+ @Deprecated
public MQClientInstance getmQClientFactory() {
return mQClientFactory;
}
diff --cc client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
index c8446bdf1,ab312f7df..02445b31f
--- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
@@@ -25,11 -36,37 +36,34 @@@ import org.apache.rocketmq.client.produ
import org.apache.rocketmq.client.producer.SendCallback;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.client.producer.SendStatus;
+ import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.PlainAccessConfig;
+ import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.protocol.RequestCode;
+ import org.apache.rocketmq.common.message.MessageDecoder;
+ import org.apache.rocketmq.common.message.MessageExt;
+ import org.apache.rocketmq.common.message.MessageQueueAssignment;
+ import org.apache.rocketmq.common.message.MessageRequestMode;
import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.body.ClusterAclVersionInfo;
+ import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody;
+ import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
-import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseBody;
+ import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody;
+ import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetEarliestMsgStoretimeResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.PopMessageResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.SearchOffsetResponseHeader;
import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;
diff --cc client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java
index a60d88ea0,26e341475..67e6b7d53
--- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java
@@@ -43,9 -44,9 +41,8 @@@ import org.mockito.junit.MockitoJUnitRu
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
- import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
diff --cc common/pom.xml
index 3304640b2,cea58306a..1ff9111db
--- a/common/pom.xml
+++ b/common/pom.xml
@@@ -40,13 -40,9 +40,17 @@@
<groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
</project>
diff --cc common/src/main/java/org/apache/rocketmq/common/MQVersion.java
index 60ab6f8cb,847fd7436..544ce5f44
--- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
+++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
@@@ -18,7 -18,7 +18,7 @@@ package org.apache.rocketmq.common
public class MQVersion {
- public static final int CURRENT_VERSION = Version.V4_9_4.ordinal();
- public static final int CURRENT_VERSION = Version.V5_0_0_BETA_SNAPSHOT.ordinal();
++ public static final int CURRENT_VERSION = Version.V5_0_0_SNAPSHOT.ordinal();
public static String getVersionDesc(int value) {
int length = Version.values().length;
diff --cc common/src/main/java/org/apache/rocketmq/common/MixAll.java
index b13a09f92,638d03806..0a6bcf1ee
--- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java
+++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
@@@ -89,8 -89,11 +93,12 @@@ public class MixAll
public static final String REPLY_MESSAGE_FLAG = "reply";
public static final String LMQ_PREFIX = "%LMQ%";
public static final String MULTI_DISPATCH_QUEUE_SPLITTER = ",";
+ public static final String REQ_T = "ReqT";
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
+ public static final String LOGICAL_QUEUE_MOCK_BROKER_PREFIX = "__syslo__";
+ public static final String METADATA_SCOPE_GLOBAL = "__global__";
+ public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST = "__syslo__none__";
+ public static final String MULTI_PATH_SPLITTER = System.getProperty("rocketmq.broker.multiPathSplitter", ",");
public static String getWSAddr() {
String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP);
diff --cc common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
index b86780f7d,d6d4a8f4f..0ff544cbb
--- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
+++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
@@@ -348,10 -470,17 +471,18 @@@ public class MessageDecoder
byte[] body = new byte[bodyLen];
byteBuffer.get(body);
+ if (checkCRC) {
+ //crc body
+ int crc = UtilAll.crc32(body, 0, bodyLen);
+ if (crc != bodyCRC) {
+ throw new Exception("Msg crc is error!");
+ }
+ }
+
// uncompress body
if (deCompressBody && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) {
- body = UtilAll.uncompress(body);
+ Compressor compressor = CompressorFactory.getCompressor(MessageSysFlag.getCompressionType(sysFlag));
+ body = compressor.decompress(body);
}
msgExt.setBody(body);
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
index 9912382ff,1b2692da0..918b34077
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
@@@ -191,7 -204,37 +204,36 @@@ public class RequestCode
public static final int ADD_WRITE_PERM_OF_BROKER = 327;
-
+ public static final int GET_TOPIC_CONFIG = 351;
+
+ public static final int GET_SUBSCRIPTIONGROUP_CONFIG = 352;
+ public static final int UPDATE_AND_GET_GROUP_FORBIDDEN = 353;
+
+ public static final int LITE_PULL_MESSAGE = 361;
+
+ public static final int QUERY_ASSIGNMENT = 400;
+ public static final int SET_MESSAGE_REQUEST_MODE = 401;
+ public static final int GET_ALL_MESSAGE_REQUEST_MODE = 402;
+
+ public static final int UPDATE_AND_CREATE_STATIC_TOPIC = 513;
+
- /**
- * Below request codes are used by broker container,
- * these request codes are started with '9'.
- */
+ public static final int GET_BROKER_MEMBER_GROUP = 901;
+
+ public static final int ADD_BROKER = 902;
+
+ public static final int REMOVE_BROKER = 903;
+
+ public static final int BROKER_HEARTBEAT = 904;
+
+ public static final int NOTIFY_MIN_BROKER_ID_CHANGE = 905;
+
+ public static final int EXCHANGE_BROKER_HA_INFO = 906;
+
+ public static final int GET_BROKER_HA_STATUS = 907;
+
+ public static final int RESET_MASTER_FLUSH_OFFSET = 908;
++
+ public static final int GET_ALL_PRODUCER_INFO = 328;
+
+ public static final int DELETE_EXPIRED_COMMITLOG = 329;
}
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
index 02fac8e59,08790fc29..486efdfb5
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
@@@ -20,17 -20,12 +20,16 @@@
*/
package org.apache.rocketmq.common.protocol.header;
+import java.util.HashMap;
-
- import org.apache.rocketmq.remoting.CommandCustomHeader;
+ import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.FastCodesHeader;
-public class PullMessageRequestHeader extends TopicQueueRequestHeader {
+import io.netty.buffer.ByteBuf;
+
- public class PullMessageRequestHeader implements CommandCustomHeader, FastCodesHeader {
++public class PullMessageRequestHeader extends TopicQueueRequestHeader implements FastCodesHeader {
@CFNotNull
private String consumerGroup;
@CFNotNull
@@@ -57,79 -55,6 +59,115 @@@
public void checkFields() throws RemotingCommandException {
}
+ @Override
+ public void encode(ByteBuf out) {
+ writeIfNotNull(out, "consumerGroup", consumerGroup);
+ writeIfNotNull(out, "topic", topic);
+ writeIfNotNull(out, "queueId", queueId);
+ writeIfNotNull(out, "queueOffset", queueOffset);
+ writeIfNotNull(out, "maxMsgNums", maxMsgNums);
+ writeIfNotNull(out, "sysFlag", sysFlag);
+ writeIfNotNull(out, "commitOffset", commitOffset);
+ writeIfNotNull(out, "suspendTimeoutMillis", suspendTimeoutMillis);
+ writeIfNotNull(out, "subscription", subscription);
+ writeIfNotNull(out, "subVersion", subVersion);
+ writeIfNotNull(out, "expressionType", expressionType);
++ writeIfNotNull(out, "maxMsgBytes", maxMsgBytes);
++ writeIfNotNull(out, "lo", lo);
++ writeIfNotNull(out, "ns", ns);
++ writeIfNotNull(out, "nsd", nsd);
++ writeIfNotNull(out, "bname", bname);
++ writeIfNotNull(out, "oway", oway);
+ }
+
+ @Override
+ public void decode(HashMap<String, String> fields) throws RemotingCommandException {
+ String str = getAndCheckNotNull(fields, "consumerGroup");
+ if (str != null) {
+ this.consumerGroup = str;
+ }
+
+ str = getAndCheckNotNull(fields, "topic");
+ if (str != null) {
+ this.topic = str;
+ }
+
+ str = getAndCheckNotNull(fields, "queueId");
+ if (str != null) {
+ this.queueId = Integer.parseInt(str);
+ }
+
+ str = getAndCheckNotNull(fields, "queueOffset");
+ if (str != null) {
+ this.queueOffset = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "maxMsgNums");
+ if (str != null) {
+ this.maxMsgNums = Integer.parseInt(str);
+ }
+
+ str = getAndCheckNotNull(fields, "sysFlag");
+ if (str != null) {
+ this.sysFlag = Integer.parseInt(str);
+ }
+
+ str = getAndCheckNotNull(fields, "commitOffset");
+ if (str != null) {
+ this.commitOffset = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "suspendTimeoutMillis");
+ if (str != null) {
+ this.suspendTimeoutMillis = Long.parseLong(str);
+ }
+
+ str = fields.get("subscription");
+ if (str != null) {
+ this.subscription = str;
+ }
+
+ str = getAndCheckNotNull(fields, "subVersion");
+ if (str != null) {
+ this.subVersion = Long.parseLong(str);
+ }
+
+ str = fields.get("expressionType");
+ if (str != null) {
+ this.expressionType = str;
+ }
++
++ str = fields.get("maxMsgBytes");
++ if (str != null) {
++ this.maxMsgBytes = Integer.parseInt(str);
++ }
++
++ str = fields.get("lo");
++ if (str != null) {
++ this.lo = Boolean.parseBoolean(str);
++ }
++
++ str = fields.get("ns");
++ if (str != null) {
++ this.ns = str;
++ }
++
++ str = fields.get("nsd");
++ if (str != null) {
++ this.nsd = Boolean.parseBoolean(str);
++ }
++
++ str = fields.get("bname");
++ if (str != null) {
++ this.bname = str;
++ }
++
++ str = fields.get("oway");
++ if (str != null) {
++ this.oway = Boolean.parseBoolean(str);
++ }
+ }
+
public String getConsumerGroup() {
return consumerGroup;
}
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java
index 580d22277,84e6daf12..fb92a72d6
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java
@@@ -20,16 -20,12 +20,17 @@@
*/
package org.apache.rocketmq.common.protocol.header;
+import java.util.HashMap;
+
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
+ import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.FastCodesHeader;
+
+import io.netty.buffer.ByteBuf;
-public class PullMessageResponseHeader implements CommandCustomHeader {
+public class PullMessageResponseHeader implements CommandCustomHeader, FastCodesHeader {
@CFNotNull
private Long suggestWhichBrokerId;
@CFNotNull
@@@ -43,37 -47,6 +52,62 @@@
public void checkFields() throws RemotingCommandException {
}
+ @Override
+ public void encode(ByteBuf out) {
+ writeIfNotNull(out, "suggestWhichBrokerId", suggestWhichBrokerId);
+ writeIfNotNull(out, "nextBeginOffset", nextBeginOffset);
+ writeIfNotNull(out, "minOffset", minOffset);
+ writeIfNotNull(out, "maxOffset", maxOffset);
++ writeIfNotNull(out, "offsetDelta", offsetDelta);
++ writeIfNotNull(out, "topicSysFlag", topicSysFlag);
++ writeIfNotNull(out, "groupSysFlag", groupSysFlag);
++ writeIfNotNull(out, "forbiddenType", forbiddenType);
+ }
+
+ @Override
+ public void decode(HashMap<String, String> fields) throws RemotingCommandException {
+ String str = getAndCheckNotNull(fields, "suggestWhichBrokerId");
+ if (str != null) {
+ this.suggestWhichBrokerId = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "nextBeginOffset");
+ if (str != null) {
+ this.nextBeginOffset = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "minOffset");
+ if (str != null) {
+ this.minOffset = Long.parseLong(str);
+ }
+
+ str = getAndCheckNotNull(fields, "maxOffset");
+ if (str != null) {
+ this.maxOffset = Long.parseLong(str);
+ }
++
++ str = getAndCheckNotNull(fields, "offsetDelta");
++ if (str != null) {
++ this.offsetDelta = Long.parseLong(str);
++ }
++
++ str = getAndCheckNotNull(fields, "topicSysFlag");
++ if (str != null) {
++ this.topicSysFlag = Integer.parseInt(str);
++ }
++
++ str = getAndCheckNotNull(fields, "groupSysFlag");
++ if (str != null) {
++ this.groupSysFlag = Integer.parseInt(str);
++ }
++
++ str = getAndCheckNotNull(fields, "forbiddenType");
++ if (str != null) {
++ this.forbiddenType = Integer.parseInt(str);
++ }
++
+ }
+
public Long getNextBeginOffset() {
return nextBeginOffset;
}
@@@ -105,4 -78,39 +139,36 @@@
public void setSuggestWhichBrokerId(Long suggestWhichBrokerId) {
this.suggestWhichBrokerId = suggestWhichBrokerId;
}
+
+ public Integer getTopicSysFlag() {
+ return topicSysFlag;
+ }
+
+ public void setTopicSysFlag(Integer topicSysFlag) {
+ this.topicSysFlag = topicSysFlag;
+ }
+
+ public Integer getGroupSysFlag() {
+ return groupSysFlag;
+ }
+
+ public void setGroupSysFlag(Integer groupSysFlag) {
+ this.groupSysFlag = groupSysFlag;
+ }
- /**
- * @return the forbiddenType
- */
++
+ public Integer getForbiddenType() {
+ return forbiddenType;
+ }
- /**
- */
++
+ public void setForbiddenType(Integer forbiddenType) {
+ this.forbiddenType = forbiddenType;
+ }
+
+ public Long getOffsetDelta() {
+ return offsetDelta;
+ }
+
+ public void setOffsetDelta(Long offsetDelta) {
+ this.offsetDelta = offsetDelta;
+ }
}
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageResponseHeader.java
index 224a5264a,601f720e4..c6eaaee6b
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageResponseHeader.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageResponseHeader.java
@@@ -42,37 -38,6 +43,43 @@@ public class SendMessageResponseHeader
public void checkFields() throws RemotingCommandException {
}
+ @Override
+ public void encode(ByteBuf out) {
+ writeIfNotNull(out, "msgId", msgId);
+ writeIfNotNull(out, "queueId", queueId);
+ writeIfNotNull(out, "queueOffset", queueOffset);
+ writeIfNotNull(out, "transactionId", transactionId);
++ writeIfNotNull(out, "batchUniqId", batchUniqId);
+ }
+
+ @Override
+ public void decode(HashMap<String, String> fields) throws RemotingCommandException {
+ String str = getAndCheckNotNull(fields, "msgId");
+ if (str != null) {
+ this.msgId = str;
+ }
+
+ str = getAndCheckNotNull(fields, "queueId");
+ if (str != null) {
+ this.queueId = Integer.parseInt(str);
+ }
+
+ str = getAndCheckNotNull(fields, "queueOffset");
+ if (str != null) {
+ this.queueOffset = Long.parseLong(str);
+ }
+
+ str = fields.get("transactionId");
+ if (str != null) {
+ this.transactionId = str;
+ }
++
++ str = fields.get("batchUniqId");
++ if (str != null) {
++ this.batchUniqId = str;
++ }
+ }
+
public String getMsgId() {
return msgId;
}
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
index 50c789363,9a67ca5dc..6bca94d8d
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
@@@ -20,9 -20,8 +20,10 @@@ package org.apache.rocketmq.common.prot
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+ import java.util.Map;
import java.util.Random;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.MixAll;
public class BrokerData implements Comparable<BrokerData> {
@@@ -42,9 -58,17 +60,17 @@@
this.brokerAddrs = brokerAddrs;
}
+ public BrokerData(String cluster, String brokerName, HashMap<Long, String> brokerAddrs,
+ boolean enableActingMaster) {
+ this.cluster = cluster;
+ this.brokerName = brokerName;
+ this.brokerAddrs = brokerAddrs;
+ this.enableActingMaster = enableActingMaster;
+ }
+
/**
-- * Selects a (preferably master) broker address from the registered list.
-- * If the master's address cannot be found, a slave broker address is selected in a random manner.
++ * Selects a (preferably master) broker address from the registered list. If the master's address cannot be found, a
++ * slave broker address is selected in a random manner.
*
* @return Broker address.
*/
@@@ -86,19 -118,28 +120,24 @@@
@Override
public boolean equals(Object obj) {
- if (this == obj)
+ if (this == obj) {
return true;
- if (obj == null)
+ }
+ if (obj == null) {
return false;
- if (getClass() != obj.getClass())
+ }
+ if (getClass() != obj.getClass()) {
return false;
+ }
BrokerData other = (BrokerData) obj;
if (brokerAddrs == null) {
- if (other.brokerAddrs != null)
+ if (other.brokerAddrs != null) {
return false;
- } else if (!brokerAddrs.equals(other.brokerAddrs))
+ }
+ } else if (!brokerAddrs.equals(other.brokerAddrs)) {
return false;
+ }
- if (brokerName == null) {
- return other.brokerName == null;
- } else {
- return brokerName.equals(other.brokerName);
- }
+ return StringUtils.equals(brokerName, other.brokerName);
}
@Override
diff --cc common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
index 2d900398e,0c02078eb..a53b8eebd
--- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
+++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
@@@ -33,21 -69,47 +69,49 @@@ public class TopicRouteData extends Rem
public TopicRouteData cloneTopicRouteData() {
TopicRouteData topicRouteData = new TopicRouteData();
-
+ topicRouteData.setQueueDatas(new ArrayList<>());
+ topicRouteData.setBrokerDatas(new ArrayList<>());
+ topicRouteData.setFilterServerTable(new HashMap<>());
topicRouteData.setOrderTopicConf(this.orderTopicConf);
- if (this.queueDatas != null) {
- topicRouteData.getQueueDatas().addAll(this.queueDatas);
+ topicRouteData.getQueueDatas().addAll(this.queueDatas);
+ topicRouteData.getBrokerDatas().addAll(this.brokerDatas);
+ topicRouteData.getFilterServerTable().putAll(this.filterServerTable);
+ if (this.topicQueueMappingByBroker != null) {
+ Map<String, TopicQueueMappingInfo> cloneMap = new HashMap<>(this.topicQueueMappingByBroker);
+ topicRouteData.setTopicQueueMappingByBroker(cloneMap);
+ }
+ return topicRouteData;
+ }
+
+ public TopicRouteData deepCloneTopicRouteData() {
+ TopicRouteData topicRouteData = new TopicRouteData();
+
+ topicRouteData.setOrderTopicConf(this.orderTopicConf);
+
+ for (final QueueData queueData : this.queueDatas) {
+ topicRouteData.getQueueDatas().add(new QueueData(queueData));
}
- if (this.brokerDatas != null) {
- topicRouteData.getBrokerDatas().addAll(this.brokerDatas);
+ for (final BrokerData brokerData : this.brokerDatas) {
+ topicRouteData.getBrokerDatas().add(new BrokerData(brokerData));
}
- if (this.filterServerTable != null) {
- topicRouteData.getFilterServerTable().putAll(this.filterServerTable);
+ for (final Map.Entry<String, List<String>> listEntry : this.filterServerTable.entrySet()) {
+ topicRouteData.getFilterServerTable().put(listEntry.getKey(),
+ new ArrayList<String>(listEntry.getValue()));
+ }
+ if (this.topicQueueMappingByBroker != null) {
+ Map<String, TopicQueueMappingInfo> cloneMap = new HashMap<>(this.topicQueueMappingByBroker.size());
+ for (final Map.Entry<String, TopicQueueMappingInfo> entry : this.getTopicQueueMappingByBroker().entrySet()) {
+ TopicQueueMappingInfo topicQueueMappingInfo = new TopicQueueMappingInfo(entry.getValue().getTopic(), entry.getValue().getTotalQueues(), entry.getValue().getBname(), entry.getValue().getEpoch());
+ topicQueueMappingInfo.setDirty(entry.getValue().isDirty());
+ topicQueueMappingInfo.setScope(entry.getValue().getScope());
+ ConcurrentMap<Integer, Integer> concurrentMap = new ConcurrentHashMap<Integer, Integer>(entry.getValue().getCurrIdMap());
+ topicQueueMappingInfo.setCurrIdMap(concurrentMap);
+ cloneMap.put(entry.getKey(), topicQueueMappingInfo);
+ }
+ topicRouteData.setTopicQueueMappingByBroker(cloneMap);
}
return topicRouteData;
diff --cc common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
index 2373cde12,d28ac621c..bf6060214
--- a/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
+++ b/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
@@@ -16,21 -16,7 +16,20 @@@
*/
package org.apache.rocketmq.common.sysflag;
+import org.apache.rocketmq.common.compression.CompressionType;
+
public class MessageSysFlag {
+
+ /**
+ * Meaning of each bit in the system flag
+ *
+ * | bit | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+ * |--------|---|---|-----------|----------|-------------|------------------|------------------|------------------|
+ * | byte 1 | | | STOREHOST | BORNHOST | TRANSACTION | TRANSACTION | MULTI_TAGS | COMPRESSED |
+ * | byte 2 | | | | | | COMPRESSION_TYPE | COMPRESSION_TYPE | COMPRESSION_TYPE |
+ * | byte 3 | | | | | | | | |
+ * | byte 4 | | | | | | | | |
- *
+ */
public final static int COMPRESSED_FLAG = 0x1;
public final static int MULTI_TAGS_FLAG = 0x1 << 1;
public final static int TRANSACTION_NOT_TYPE = 0;
@@@ -61,9 -40,7 +60,13 @@@
return flag & (~COMPRESSED_FLAG);
}
+ // To match the compression type
+ public static CompressionType getCompressionType(final int flag) {
+ return CompressionType.findByValue((flag & COMPRESSION_TYPE_COMPARATOR) >> 8);
+ }
+
+ public static boolean check(int flag, int expectedFlag) {
+ return (flag & expectedFlag) != 0;
+ }
++
}
diff --cc common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
index 0448ceb11,72857bc4a..83703fc10
--- a/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
@@@ -14,11 -14,12 +14,13 @@@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
- package org.apache.rocketmq.broker.util;
+
+ package org.apache.rocketmq.common.utils;
+import java.nio.charset.StandardCharsets;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
+ import org.apache.rocketmq.common.constant.LoggerName;
+ import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.logging.InternalLoggerFactory;
import java.io.BufferedReader;
import java.io.InputStream;
@@@ -36,20 -36,22 +37,19 @@@ public class ServiceProvider
private static ClassLoader thisClassLoader;
/**
-- * JDK1.3+ <a href= "http://java.sun.com/j2se/1.3/docs/guide/jar/jar.html#Service%20Provider" > 'Service Provider' specification</a>.
++ * JDK1.3+ <a href= "http://java.sun.com/j2se/1.3/docs/guide/jar/jar.html#Service%20Provider" > 'Service Provider'
++ * specification</a>.
*/
public static final String TRANSACTION_SERVICE_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService";
public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener";
+ public static final String HA_SERVICE_ID = "META-INF/service/org.apache.rocketmq.store.ha.HAService";
-
public static final String RPC_HOOK_ID = "META-INF/service/org.apache.rocketmq.remoting.RPCHook";
--
public static final String ACL_VALIDATOR_ID = "META-INF/service/org.apache.rocketmq.acl.AccessValidator";
--
--
static {
thisClassLoader = getClassLoader(ServiceProvider.class);
}
@@@ -57,7 -59,7 +57,8 @@@
/**
* Returns a string that uniquely identifies the specified object, including its class.
* <p>
-- * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString() method, but works even when the specified object's class has overidden the toString method.
++ * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString()
++ * method, but works even when the specified object's class has overidden the toString method.
*
* @param o may be null.
* @return a string of form classname@hashcode, or "null" if param o is null.
@@@ -120,7 -126,7 +121,7 @@@
names.add(serviceName);
}
-- services.add((T)initService(getContextClassLoader(), serviceName, clazz));
++ services.add((T) initService(getContextClassLoader(), serviceName, clazz));
serviceName = reader.readLine();
}
@@@ -170,10 -180,10 +171,10 @@@
// This indicates a problem with the ClassLoader tree. An incompatible ClassLoader was used to load the implementation.
LOG.error(
"Class {} loaded from classloader {} does not extend {} as loaded by this classloader.",
- new Object[] {serviceClazz.getName(),
- objectId(serviceClazz.getClassLoader()), clazz.getName()});
+ serviceClazz.getName(),
+ objectId(serviceClazz.getClassLoader()), clazz.getName());
}
- return (T)serviceClazz.getDeclaredConstructor().newInstance();
- return (T)serviceClazz.newInstance();
++ return (T) serviceClazz.getDeclaredConstructor().newInstance();
} catch (ClassNotFoundException ex) {
if (classLoader == thisClassLoader) {
// Nothing more to try, onwards.
@@@ -196,6 -206,6 +197,6 @@@
} catch (Exception e) {
LOG.error("Unable to init service.", e);
}
-- return (T)serviceClazz;
++ return (T) serviceClazz;
}
}
diff --cc common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java
index 1734cbdf7,1734cbdf7..6b0f75acb
--- a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java
+++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java
@@@ -22,8 -22,8 +22,6 @@@ import org.junit.Test
import static org.assertj.core.api.Assertions.assertThat;
--import java.nio.charset.StandardCharsets;
--
public class MessageClientIDSetterTest {
@Test
diff --cc container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
index 000000000,2512a3307..c684f0eb8
mode 000000,100644..100644
--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
+++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
@@@ -1,0 -1,280 +1,280 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.rocketmq.container;
+
+ import io.netty.channel.ChannelHandlerContext;
+
+ import java.io.UnsupportedEncodingException;
+ import java.util.List;
+ import java.util.Properties;
+ import org.apache.rocketmq.broker.BrokerController;
+ import org.apache.rocketmq.broker.BrokerStartup;
+ import org.apache.rocketmq.common.BrokerConfig;
+ import org.apache.rocketmq.common.BrokerIdentity;
+ import org.apache.rocketmq.common.MixAll;
+ import org.apache.rocketmq.common.constant.LoggerName;
+ import org.apache.rocketmq.common.protocol.RequestCode;
+ import org.apache.rocketmq.common.protocol.ResponseCode;
+ import org.apache.rocketmq.common.protocol.header.AddBrokerRequestHeader;
+ import org.apache.rocketmq.common.protocol.header.GetBrokerConfigResponseHeader;
+ import org.apache.rocketmq.common.protocol.header.RemoveBrokerRequestHeader;
+ import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.logging.InternalLoggerFactory;
+ import org.apache.rocketmq.remoting.common.RemotingHelper;
+ import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+ import org.apache.rocketmq.store.config.MessageStoreConfig;
+
+ public class BrokerContainerProcessor implements NettyRequestProcessor {
+ private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+ private final BrokerContainer brokerContainer;
+ private List<BrokerBootHook> brokerBootHookList;
+
+ public BrokerContainerProcessor(BrokerContainer brokerContainer) {
+ this.brokerContainer = brokerContainer;
+ }
+
+ @Override
+ public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {
+ switch (request.getCode()) {
+ case RequestCode.ADD_BROKER:
+ return this.addBroker(ctx, request);
+ case RequestCode.REMOVE_BROKER:
+ return this.removeBroker(ctx, request);
+ case RequestCode.GET_BROKER_CONFIG:
+ return this.getBrokerConfig(ctx, request);
+ case RequestCode.UPDATE_BROKER_CONFIG:
+ return this.updateBrokerConfig(ctx, request);
+ default:
+ break;
+ }
+ return null;
+ }
+
+ @Override public boolean rejectRequest() {
+ return false;
+ }
+
+ private synchronized RemotingCommand addBroker(ChannelHandlerContext ctx,
+ RemotingCommand request) throws Exception {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final AddBrokerRequestHeader requestHeader = request.decodeCommandCustomHeader(AddBrokerRequestHeader.class);
++ final AddBrokerRequestHeader requestHeader = (AddBrokerRequestHeader) request.decodeCommandCustomHeader(AddBrokerRequestHeader.class);
+
+ LOGGER.info("addBroker called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ Properties brokerProperties = null;
+ String configPath = requestHeader.getConfigPath();
+
+ if (configPath != null && !configPath.isEmpty()) {
+ BrokerStartup.SystemConfigFileHelper configFileHelper = new BrokerStartup.SystemConfigFileHelper();
+ configFileHelper.setFile(configPath);
+
+ try {
+ brokerProperties = configFileHelper.loadConfig();
+ } catch (Exception e) {
+ LOGGER.error("addBroker load config from {} failed, {}", configPath, e);
+ }
+ } else {
+ byte[] body = request.getBody();
+ if (body != null) {
+ String bodyStr = new String(body, MixAll.DEFAULT_CHARSET);
+ brokerProperties = MixAll.string2Properties(bodyStr);
+ }
+ }
+
+ if (brokerProperties == null) {
+ LOGGER.error("addBroker properties empty");
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("addBroker properties empty");
+ return response;
+ }
+
+ BrokerConfig brokerConfig = new BrokerConfig();
+ MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+ MixAll.properties2Object(brokerProperties, brokerConfig);
+ MixAll.properties2Object(brokerProperties, messageStoreConfig);
+
+ messageStoreConfig.setHaListenPort(brokerConfig.getListenPort() + 1);
+
+ if (configPath != null && !configPath.isEmpty()) {
+ brokerConfig.setBrokerConfigPath(configPath);
+ }
+
+ if (!messageStoreConfig.isEnableDLegerCommitLog()) {
+ switch (messageStoreConfig.getBrokerRole()) {
+ case ASYNC_MASTER:
+ case SYNC_MASTER:
+ brokerConfig.setBrokerId(MixAll.MASTER_ID);
+ break;
+ case SLAVE:
+ if (brokerConfig.getBrokerId() <= 0) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("slave broker id must be > 0");
+ return response;
+ }
+ break;
+ default:
+ break;
+
+ }
+
+ if (messageStoreConfig.getTotalReplicas() < messageStoreConfig.getInSyncReplicas()
+ || messageStoreConfig.getTotalReplicas() < messageStoreConfig.getMinInSyncReplicas()
+ || messageStoreConfig.getInSyncReplicas() < messageStoreConfig.getMinInSyncReplicas()) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("invalid replicas number");
+ return response;
+ }
+ }
+
+ BrokerController brokerController;
+ try {
+ brokerController = this.brokerContainer.addBroker(brokerConfig, messageStoreConfig);
+ } catch (Exception e) {
+ LOGGER.error("addBroker exception {}", e);
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(e.getMessage());
+ return response;
+ }
+ if (brokerController != null) {
+ brokerController.getConfiguration().registerConfig(brokerProperties);
+ try {
+ for (BrokerBootHook brokerBootHook : brokerBootHookList) {
+ brokerBootHook.executeBeforeStart(brokerController, brokerProperties);
+ }
+ brokerController.start();
+
+ for (BrokerBootHook brokerBootHook : brokerBootHookList) {
+ brokerBootHook.executeAfterStart(brokerController, brokerProperties);
+ }
+ } catch (Exception e) {
+ LOGGER.error("start broker exception {}", e);
+ BrokerIdentity brokerIdentity;
+ if (messageStoreConfig.isEnableDLegerCommitLog()) {
+ brokerIdentity = new BrokerIdentity(brokerConfig.getBrokerClusterName(),
+ brokerConfig.getBrokerName(), Integer.parseInt(messageStoreConfig.getdLegerSelfId().substring(1)));
+ } else {
+ brokerIdentity = new BrokerIdentity(brokerConfig.getBrokerClusterName(),
+ brokerConfig.getBrokerName(), brokerConfig.getBrokerId());
+ }
+ this.brokerContainer.removeBroker(brokerIdentity);
+ brokerController.shutdown();
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("start broker failed, " + e);
+ return response;
+ }
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("add broker return null");
+ }
+
+ return response;
+ }
+
+ private synchronized RemotingCommand removeBroker(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final RemoveBrokerRequestHeader requestHeader = request.decodeCommandCustomHeader(RemoveBrokerRequestHeader.class);
++ final RemoveBrokerRequestHeader requestHeader = (RemoveBrokerRequestHeader) request.decodeCommandCustomHeader(RemoveBrokerRequestHeader.class);
+
+ LOGGER.info("removeBroker called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ BrokerIdentity brokerIdentity = new BrokerIdentity(requestHeader.getBrokerClusterName(), requestHeader.getBrokerName(), requestHeader.getBrokerId());
+
+ BrokerController brokerController;
+ try {
+ brokerController = this.brokerContainer.removeBroker(brokerIdentity);
+ } catch (Exception e) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(e.getMessage());
+ return response;
+ }
+
+ if (brokerController != null) {
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.BROKER_NOT_EXIST);
+ response.setRemark("Broker not exist");
+ }
+ return response;
+ }
+
+ public void registerBrokerBootHook(List<BrokerBootHook> brokerBootHookList) {
+ this.brokerBootHookList = brokerBootHookList;
+ }
+
+ private RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ LOGGER.info("updateSharedBrokerConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ byte[] body = request.getBody();
+ if (body != null) {
+ try {
+ String bodyStr = new String(body, MixAll.DEFAULT_CHARSET);
+ Properties properties = MixAll.string2Properties(bodyStr);
+ if (properties != null) {
+ LOGGER.info("updateSharedBrokerConfig, new config: [{}] client: {} ", properties, ctx.channel().remoteAddress());
+ this.brokerContainer.getConfiguration().update(properties);
+ } else {
+ LOGGER.error("string2Properties error");
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("string2Properties error");
+ return response;
+ }
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.error("", e);
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("UnsupportedEncodingException " + e);
+ return response;
+ }
+ }
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
+
+ private RemotingCommand getBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {
+
+ final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerConfigResponseHeader.class);
+ final GetBrokerConfigResponseHeader responseHeader = (GetBrokerConfigResponseHeader) response.readCustomHeader();
+
+ String content = this.brokerContainer.getConfiguration().getAllConfigsFormatString();
+ if (content != null && content.length() > 0) {
+ try {
+ response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.error("", e);
+
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("UnsupportedEncodingException " + e);
+ return response;
+ }
+ }
+
+ responseHeader.setVersion(this.brokerContainer.getConfiguration().getDataVersionJson());
+
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ return response;
+ }
+ }
diff --cc namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
index 94797667d,5c9a9340d..09069eaf8
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
@@@ -517,40 -529,62 +529,61 @@@ public class DefaultRequestProcessor im
}
private RemotingCommand getUnitTopicList(ChannelHandlerContext ctx,
- RemotingCommand request) throws RemotingCommandException {
+ RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- TopicList unitTopics = this.namesrvController.getRouteInfoManager().getUnitTopics();
- byte[] body = unitTopics.encode();
+ boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();
+
+ if (enableTopicList) {
+ TopicList unitTopicList = this.namesrvController.getRouteInfoManager().getUnitTopics();
+ byte[] body = unitTopicList.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("disable");
+ }
- response.setBody(body);
- response.setCode(ResponseCode.SUCCESS);
- response.setRemark(null);
return response;
}
private RemotingCommand getHasUnitSubTopicList(ChannelHandlerContext ctx,
- RemotingCommand request) throws RemotingCommandException {
+ RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- TopicList hasUnitSubTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubTopicList();
- byte[] body = hasUnitSubTopicList.encode();
+ boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();
+
+ if (enableTopicList) {
+ TopicList hasUnitSubTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubTopicList();
+ byte[] body = hasUnitSubTopicList.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("disable");
+ }
- response.setBody(body);
- response.setCode(ResponseCode.SUCCESS);
- response.setRemark(null);
return response;
}
- private RemotingCommand getHasUnitSubUnUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request)
- throws RemotingCommandException {
+ private RemotingCommand getHasUnitSubUnUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- TopicList hasUnitSubUnUnitTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubUnUnitTopicList();
- byte[] body = hasUnitSubUnUnitTopicList.encode();
+ boolean enableTopicList = namesrvController.getNamesrvConfig().isEnableTopicList();
+
+ if (enableTopicList) {
+ TopicList hasUnitSubUnUnitTopicList = this.namesrvController.getRouteInfoManager().getHasUnitSubUnUnitTopicList();
+ byte[] body = hasUnitSubUnUnitTopicList.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("disable");
+ }
- response.setBody(body);
- response.setCode(ResponseCode.SUCCESS);
- response.setRemark(null);
return response;
}
diff --cc namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
index 400b7e743,aac1259ea..4b657627b
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
@@@ -52,20 -65,47 +65,47 @@@ import org.apache.rocketmq.remoting.pro
public class RouteInfoManager {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);
- private final static long BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2;
+ private final static long DEFAULT_BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2;
private final ReadWriteLock lock = new ReentrantReadWriteLock();
- private final HashMap<String/* topic */, Map<String /* brokerName */ , QueueData>> topicQueueTable;
- private final HashMap<String/* brokerName */, BrokerData> brokerAddrTable;
- private final HashMap<String/* clusterName */, Set<String/* brokerName */>> clusterAddrTable;
- private final HashMap<String/* brokerAddr */, BrokerLiveInfo> brokerLiveTable;
- private final HashMap<String/* brokerAddr */, List<String>/* Filter Server */> filterServerTable;
+ private final Map<String/* topic */, Map<String, QueueData>> topicQueueTable;
+ private final Map<String/* brokerName */, BrokerData> brokerAddrTable;
+ private final Map<String/* clusterName */, Set<String/* brokerName */>> clusterAddrTable;
+ private final Map<BrokerAddrInfo/* brokerAddr */, BrokerLiveInfo> brokerLiveTable;
+ private final Map<BrokerAddrInfo/* brokerAddr */, List<String>/* Filter Server */> filterServerTable;
+ private final Map<String/* topic */, Map<String/*brokerName*/, TopicQueueMappingInfo>> topicQueueMappingInfoTable;
+
+ private final BatchUnRegisterService unRegisterService;
+
+ private final NamesrvController namesrvController;
+ private final NamesrvConfig namesrvConfig;
+
+ public RouteInfoManager(final NamesrvConfig namesrvConfig, NamesrvController namesrvController) {
- this.topicQueueTable = new ConcurrentHashMap<String, Map<String, QueueData>>(1024);
- this.brokerAddrTable = new ConcurrentHashMap<String, BrokerData>(128);
- this.clusterAddrTable = new ConcurrentHashMap<String, Set<String>>(32);
- this.brokerLiveTable = new ConcurrentHashMap<BrokerAddrInfo, BrokerLiveInfo>(256);
- this.filterServerTable = new ConcurrentHashMap<BrokerAddrInfo, List<String>>(256);
- this.topicQueueMappingInfoTable = new ConcurrentHashMap<String, Map<String, TopicQueueMappingInfo>>(1024);
++ this.topicQueueTable = new ConcurrentHashMap<>(1024);
++ this.brokerAddrTable = new ConcurrentHashMap<>(128);
++ this.clusterAddrTable = new ConcurrentHashMap<>(32);
++ this.brokerLiveTable = new ConcurrentHashMap<>(256);
++ this.filterServerTable = new ConcurrentHashMap<>(256);
++ this.topicQueueMappingInfoTable = new ConcurrentHashMap<>(1024);
+ this.unRegisterService = new BatchUnRegisterService(this, namesrvConfig);
+ this.namesrvConfig = namesrvConfig;
+ this.namesrvController = namesrvController;
+ }
+
+ public void start() {
+ this.unRegisterService.start();
+ }
- public RouteInfoManager() {
- this.topicQueueTable = new HashMap<>(1024);
- this.brokerAddrTable = new HashMap<>(128);
- this.clusterAddrTable = new HashMap<>(32);
- this.brokerLiveTable = new HashMap<>(256);
- this.filterServerTable = new HashMap<>(256);
+ public void shutdown() {
+ this.unRegisterService.shutdown(true);
+ }
+
+ public boolean submitUnRegisterBrokerRequest(UnRegisterBrokerRequestHeader unRegisterRequest) {
+ return this.unRegisterService.submit(unRegisterRequest);
+ }
+
+ // For test only
+ int blockedUnRegisterRequests() {
+ return this.unRegisterService.queueLength();
}
public ClusterInfo getAllClusterInfo() {
@@@ -96,20 -167,19 +167,19 @@@
if (brokerNames != null
&& !brokerNames.isEmpty()) {
Map<String, QueueData> queueDataMap = this.topicQueueTable.get(topic);
- for (String brokerName : brokerNames) {
- final QueueData removedQD = queueDataMap.remove(brokerName);
- if (removedQD != null) {
- log.info("deleteTopic, remove one broker's topic {} {} {}", brokerName, topic,
- removedQD);
+ if (queueDataMap != null) {
+ for (String brokerName : brokerNames) {
+ final QueueData removedQD = queueDataMap.remove(brokerName);
+ if (removedQD != null) {
+ log.info("deleteTopic, remove one broker's topic {} {} {}", brokerName, topic,
+ removedQD);
+ }
}
-
if (queueDataMap.isEmpty()) {
- log.info("deleteTopic, remove the topic all queue {} {}", brokerName, topic);
+ log.info("deleteTopic, remove the topic all queue {} {}", clusterName, topic);
this.topicQueueTable.remove(topic);
- break;
}
}
-
}
} finally {
this.lock.writeLock().unlock();
diff --cc pom.xml
index b7b7a50e6,5f6c4ed70..acb9c7049
--- a/pom.xml
+++ b/pom.xml
@@@ -98,58 -98,6 +98,60 @@@
<!-- Compiler settings properties -->
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
+
+ <slf4j.version>1.7.7</slf4j.version>
+ <logback.version>1.2.10</logback.version>
- <commons-cli.version>1.2</commons-cli.version>
++ <commons-cli.version>1.4</commons-cli.version>
+ <netty.version>4.1.65.Final</netty.version>
+ <bcpkix-jdk15on.version>1.69</bcpkix-jdk15on.version>
+ <fastjson.version>1.2.69_noneautotype</fastjson.version>
+ <javassist.version>3.20.0-GA</javassist.version>
+ <jna.version>4.2.2</jna.version>
+ <commons-lang3.version>3.4</commons-lang3.version>
++ <commons-io.version>2.6</commons-io.version>
+ <guava.version>31.0.1-jre</guava.version>
+ <openmessaging.version>0.3.1-alpha</openmessaging.version>
+ <log4j.version>1.2.17</log4j.version>
+ <snakeyaml.version>1.30</snakeyaml.version>
+ <commons-codec.version>1.9</commons-codec.version>
+ <logging-log4j.version>2.17.1</logging-log4j.version>
+ <commons-validator.version>1.7</commons-validator.version>
+ <zstd-jni.version>1.5.2-2</zstd-jni.version>
+ <lz4-java.version>1.8.0</lz4-java.version>
+ <opentracing.version>0.33.0</opentracing.version>
+ <jaeger.version>1.6.0</jaeger.version>
+ <dleger.version>0.2.6</dleger.version>
+ <annotations-api.version>6.0.53</annotations-api.version>
+ <extra-enforcer-rules.version>1.0-beta-4</extra-enforcer-rules.version>
++ <concurrentlinkedhashmap-lru.version>1.4.2</concurrentlinkedhashmap-lru.version>
+
+ <!-- Test dependencies -->
+ <junit.version>4.13.2</junit.version>
+ <assertj-core.version>2.6.0</assertj-core.version>
+ <mockito-core.version>3.10.0</mockito-core.version>
+ <awaitility.version>4.1.0</awaitility.version>
+ <truth.version>0.30</truth.version>
+
+ <!-- Build plugin dependencies -->
+ <versions-maven-plugin.version>2.2</versions-maven-plugin.version>
+ <dependency-mediator-maven-plugin.version>1.0.2</dependency-mediator-maven-plugin.version>
+ <clirr-maven-plugin.version>2.7</clirr-maven-plugin.version>
+ <maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>
+ <maven-compiler-plugin.version>3.5.1</maven-compiler-plugin.version>
+ <maven-source-plugin.version>3.0.1</maven-source-plugin.version>
+ <maven-help-plugin.version>2.2</maven-help-plugin.version>
+ <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
+ <apache-rat-plugin.version>0.12</apache-rat-plugin.version>
+ <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>
+ <coveralls-maven-plugin.version>4.3.0</coveralls-maven-plugin.version>
+ <jacoco-maven-plugin.version>0.8.5</jacoco-maven-plugin.version>
+ <maven-surefire-plugin.version>2.19.1</maven-surefire-plugin.version>
+ <findbugs-maven-plugin.version>3.0.4</findbugs-maven-plugin.version>
+ <sonar-maven-plugin.version>3.0.2</sonar-maven-plugin.version>
+ <maven-assembly-plugin.version>3.0.0</maven-assembly-plugin.version>
+ <maven-javadoc-plugin.version>2.10.4</maven-javadoc-plugin.version>
+ <maven-failsafe-plugin.version>2.19.1</maven-failsafe-plugin.version>
+
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
<!-- Exclude all generated code -->
<sonar.jacoco.itReportPath>${project.basedir}/../test/target/jacoco-it.exec</sonar.jacoco.itReportPath>
@@@ -588,13 -569,23 +596,23 @@@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.4</version>
+ <version>${commons-lang3.version}</version>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
- <version>2.6</version>
++ <version>${commons-io.version}</version>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>19.0</version>
+ <version>${guava.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.googlecode.concurrentlinkedhashmap</groupId>
+ <artifactId>concurrentlinkedhashmap-lru</artifactId>
- <version>1.4.2</version>
++ <version>${concurrentlinkedhashmap-lru.version}</version>
+ </dependency>
<dependency>
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-api</artifactId>
diff --cc remoting/src/main/java/org/apache/rocketmq/remoting/protocol/FastCodesHeader.java
index 760040685,000000000..ebf993088
mode 100644,000000..100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/FastCodesHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/FastCodesHeader.java
@@@ -1,50 -1,0 +1,50 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.remoting.protocol;
+
+import java.util.HashMap;
+
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+
+import io.netty.buffer.ByteBuf;
+
+public interface FastCodesHeader {
+
+ default String getAndCheckNotNull(HashMap<String, String> fields, String field) {
+ String value = fields.get(field);
+ if (value == null) {
+ String headerClass = this.getClass().getSimpleName();
+ RemotingCommand.log.error("the custom field {}.{} is null", headerClass, field);
+ // no exception throws, keep compatible with RemotingCommand.decodeCommandCustomHeader
+ }
+ return value;
+ }
+
+ default void writeIfNotNull(ByteBuf out, String key, Object value) {
+ if (value != null) {
+ RocketMQSerializable.writeStr(out, true, key);
+ RocketMQSerializable.writeStr(out, false, value.toString());
+ }
+ }
+
- public void encode(ByteBuf out);
++ void encode(ByteBuf out);
+
+ void decode(HashMap<String, String> fields) throws RemotingCommandException;
+
+
+}
diff --cc remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
index b6523a3fb,7d614cffb..720beb030
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
@@@ -17,24 -17,24 +17,28 @@@
package org.apache.rocketmq.remoting.protocol;
import com.alibaba.fastjson.annotation.JSONField;
+ import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.logging.InternalLoggerFactory;
+ import org.apache.rocketmq.remoting.CommandCustomHeader;
+ import org.apache.rocketmq.remoting.annotation.CFNotNull;
+ import org.apache.rocketmq.remoting.common.RemotingHelper;
+ import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.nio.ByteBuffer;
+ import java.util.Arrays;
import java.util.HashMap;
+ import java.util.HashSet;
import java.util.Map;
+ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
- import org.apache.rocketmq.logging.InternalLogger;
- import org.apache.rocketmq.logging.InternalLoggerFactory;
- import org.apache.rocketmq.remoting.CommandCustomHeader;
- import org.apache.rocketmq.remoting.annotation.CFNotNull;
- import org.apache.rocketmq.remoting.common.RemotingHelper;
- import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
public class RemotingCommand {
public static final String SERIALIZE_TYPE_PROPERTY = "rocketmq.serialize.type";
public static final String SERIALIZE_TYPE_ENV = "ROCKETMQ_SERIALIZE_TYPE";
@@@ -114,6 -123,17 +127,18 @@@
return createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, "not set any response code", classHeader);
}
- public static RemotingCommand buildErrorResponse(int code, String remark, Class<? extends CommandCustomHeader> classHeader) {
++ public static RemotingCommand buildErrorResponse(int code, String remark,
++ Class<? extends CommandCustomHeader> classHeader) {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(classHeader);
+ response.setCode(code);
+ response.setRemark(remark);
+ return response;
+ }
+
+ public static RemotingCommand buildErrorResponse(int code, String remark) {
+ return buildErrorResponse(code, remark, null);
+ }
+
public static RemotingCommand createResponseCommand(int code, String remark,
Class<? extends CommandCustomHeader> classHeader) {
RemotingCommand cmd = new RemotingCommand();
@@@ -178,11 -190,9 +203,12 @@@
return length & 0xFFFFFF;
}
- private static RemotingCommand headerDecode(ByteBuf byteBuffer, int len, SerializeType type) throws RemotingCommandException {
- private static RemotingCommand headerDecode(byte[] headerData, SerializeType type) throws RemotingCommandException {
++ private static RemotingCommand headerDecode(ByteBuf byteBuffer, int len,
++ SerializeType type) throws RemotingCommandException {
switch (type) {
case JSON:
+ byte[] headerData = new byte[len];
+ byteBuffer.readBytes(headerData);
RemotingCommand resultJson = RemotingSerializable.decode(headerData, RemotingCommand.class);
resultJson.setSerializeTypeCurrentRPC(type);
return resultJson;
@@@ -239,16 -255,10 +265,16 @@@
this.customHeader = customHeader;
}
- public <T extends CommandCustomHeader> T decodeCommandCustomHeader(Class<T> classHeader) throws RemotingCommandException {
- T objectHeader;
+ public CommandCustomHeader decodeCommandCustomHeader(
- Class<? extends CommandCustomHeader> classHeader) throws RemotingCommandException {
++ Class<? extends CommandCustomHeader> classHeader) throws RemotingCommandException {
+ return decodeCommandCustomHeader(classHeader, true);
+ }
+
+ public CommandCustomHeader decodeCommandCustomHeader(Class<? extends CommandCustomHeader> classHeader,
- boolean useFastEncode) throws RemotingCommandException {
++ boolean useFastEncode) throws RemotingCommandException {
+ CommandCustomHeader objectHeader;
try {
- objectHeader = classHeader.newInstance();
+ objectHeader = classHeader.getDeclaredConstructor().newInstance();
} catch (InstantiationException e) {
return null;
} catch (IllegalAccessException e) {
diff --cc remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java
index 5ec7ac902,f7ef585d5..5617b2246
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java
@@@ -19,6 -19,11 +19,9 @@@ package org.apache.rocketmq.remoting.pr
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
+ import java.util.HashSet;
+ import java.util.Set;
+
-import com.alibaba.fastjson.JSON;
-import java.util.HashSet;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
diff --cc srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
index c2aaf9620,adf61adbd..d2c77c370
--- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
+++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
@@@ -23,9 -24,9 +24,8 @@@ import java.nio.file.Files
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
- import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
diff --cc store/src/main/java/org/apache/rocketmq/store/CommitLog.java
index efed87f67,62e9f59b7..5bb6dbaad
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
@@@ -16,13 -16,30 +16,30 @@@
*/
package org.apache.rocketmq.store;
-import java.net.Inet4Address;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.UnpooledByteBufAllocator;
+ import java.net.Inet6Address;
-import java.net.InetAddress;
+ import java.net.InetSocketAddress;
-import java.net.SocketAddress;
+ import java.nio.ByteBuffer;
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.LinkedList;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Optional;
+ import java.util.Set;
+ import java.util.concurrent.CompletableFuture;
+ import java.util.concurrent.ExecutionException;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.TimeoutException;
+ import java.util.function.Supplier;
+
+ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.ServiceThread;
+ import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
- import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
@@@ -76,34 -79,31 +79,31 @@@ public class CommitLog implements Swapp
protected final PutMessageLock putMessageLock;
+ protected final TopicQueueLock topicQueueLock;
+
private volatile Set<String> fullStorePaths = Collections.emptySet();
- protected final MultiDispatch multiDispatch;
private final FlushDiskWatcher flushDiskWatcher;
- public CommitLog(final DefaultMessageStore defaultMessageStore) {
- String storePath = defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog();
- if (storePath.contains(MessageStoreConfig.MULTI_PATH_SPLITTER)) {
- this.mappedFileQueue = new MultiPathMappedFileQueue(defaultMessageStore.getMessageStoreConfig(),
- defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
- defaultMessageStore.getAllocateMappedFileService(), this::getFullStorePaths);
+ protected int commitLogSize;
+
+ public CommitLog(final DefaultMessageStore messageStore) {
+ String storePath = messageStore.getMessageStoreConfig().getStorePathCommitLog();
+ if (storePath.contains(MixAll.MULTI_PATH_SPLITTER)) {
+ this.mappedFileQueue = new MultiPathMappedFileQueue(messageStore.getMessageStoreConfig(),
+ messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
+ messageStore.getAllocateMappedFileService(), this::getFullStorePaths);
} else {
this.mappedFileQueue = new MappedFileQueue(storePath,
- defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
- defaultMessageStore.getAllocateMappedFileService());
+ messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
+ messageStore.getAllocateMappedFileService());
}
- this.defaultMessageStore = defaultMessageStore;
-
- if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
- this.flushCommitLogService = new GroupCommitService();
- } else {
- this.flushCommitLogService = new FlushRealTimeService();
- }
+ this.defaultMessageStore = messageStore;
- this.commitLogService = new CommitRealTimeService();
+ this.flushManager = new DefaultFlushManager();
- this.appendMessageCallback = new DefaultAppendMessageCallback(messageStore.getMessageStoreConfig().getMaxMessageSize());
+ this.appendMessageCallback = new DefaultAppendMessageCallback();
putMessageThreadLocal = new ThreadLocal<PutMessageThreadLocal>() {
@Override
protected PutMessageThreadLocal initialValue() {
@@@ -125,12 -127,13 +127,17 @@@
return fullStorePaths;
}
+ public long getTotalSize() {
+ return this.mappedFileQueue.getTotalFileSize();
+ }
+
+ public ThreadLocal<PutMessageThreadLocal> getPutMessageThreadLocal() {
+ return putMessageThreadLocal;
+ }
+
public boolean load() {
boolean result = this.mappedFileQueue.load();
+ this.mappedFileQueue.checkSelf();
log.info("load commit log " + (result ? "OK" : "Failed"));
return result;
}
@@@ -604,18 -754,17 +758,26 @@@
return keyBuilder.toString();
}
+ public void setMappedFileQueueOffset(final long phyOffset) {
+ this.mappedFileQueue.setFlushedWhere(phyOffset);
+ this.mappedFileQueue.setCommittedWhere(phyOffset);
+ }
+
+ public void updateMaxMessageSize(PutMessageThreadLocal putMessageThreadLocal) {
+ // dynamically adjust maxMessageSize, but not support DLedger mode temporarily
+ int newMaxMessageSize = this.defaultMessageStore.getMessageStoreConfig().getMaxMessageSize();
+ if (newMaxMessageSize >= 10 &&
+ putMessageThreadLocal.getEncoder().getMaxMessageBodySize() != newMaxMessageSize) {
+ putMessageThreadLocal.getEncoder().updateEncoderBufferCapacity(newMaxMessageSize);
+ }
+ }
+
public CompletableFuture<PutMessageResult> asyncPutMessage(final MessageExtBrokerInner msg) {
// Set the storage time
- msg.setStoreTimestamp(System.currentTimeMillis());
+ if (!defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {
+ msg.setStoreTimestamp(System.currentTimeMillis());
+ }
+
// Set the message body BODY CRC (consider the most appropriate setting
// on the client)
msg.setBodyCRC(UtilAll.crc32(msg.getBody()));
@@@ -659,65 -786,110 +799,111 @@@
}
PutMessageThreadLocal putMessageThreadLocal = this.putMessageThreadLocal.get();
+ updateMaxMessageSize(putMessageThreadLocal);
- if (!multiDispatch.isMultiDispatchMsg(msg)) {
+ String topicQueueKey = generateKey(putMessageThreadLocal.getKeyBuilder(), msg);
+ long elapsedTimeInLock = 0;
+ MappedFile unlockMappedFile = null;
+ MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
+
+ long currOffset;
+ if (mappedFile == null) {
+ currOffset = 0;
+ } else {
+ currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();
+ }
+
+ boolean needHandleHA = needHandleHA(msg);
+ int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();
+
+ if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableSlaveActingMaster()) {
+ int inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),
+ this.defaultMessageStore.getHaService().inSyncSlaveNums(currOffset) + 1);
+ needAckNums = calcNeedAckNums(inSyncReplicas);
+ if (needAckNums > inSyncReplicas) {
+ // Tell the producer, don't have enough slaves to handle the send request
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));
+ }
+ }
+
+ topicQueueLock.lock(topicQueueKey);
+ try {
+
+ boolean needAssignOffset = true;
+ if (defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()
+ && defaultMessageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) {
+ needAssignOffset = false;
+ }
+ if (needAssignOffset) {
+ defaultMessageStore.assignOffset(msg, getMessageNum(msg));
+ }
+
PutMessageResult encodeResult = putMessageThreadLocal.getEncoder().encode(msg);
if (encodeResult != null) {
return CompletableFuture.completedFuture(encodeResult);
}
- msg.setEncodedBuff(putMessageThreadLocal.getEncoder().encoderBuffer);
+ msg.setEncodedBuff(putMessageThreadLocal.getEncoder().getEncoderBuffer());
- }
- PutMessageContext putMessageContext = new PutMessageContext(generateKey(putMessageThreadLocal.getKeyBuilder(), msg));
+ PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);
- long elapsedTimeInLock = 0;
- MappedFile unlockMappedFile = null;
+ putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
+ try {
+ long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
+ this.beginTimeInLock = beginLockTimestamp;
- putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
- try {
- MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
- long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
- this.beginTimeInLock = beginLockTimestamp;
+ // Here settings are stored timestamp, in order to ensure an orderly
+ // global
+ if (!defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {
+ msg.setStoreTimestamp(beginLockTimestamp);
+ }
- // Here settings are stored timestamp, in order to ensure an orderly
- // global
- msg.setStoreTimestamp(beginLockTimestamp);
+ if (null == mappedFile || mappedFile.isFull()) {
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
+ }
+ if (null == mappedFile) {
+ log.error("create mapped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null));
+ }
- if (null == mappedFile || mappedFile.isFull()) {
- mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
- }
- if (null == mappedFile) {
- log.error("create mapped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null));
- }
+ result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
+ switch (result.getStatus()) {
+ case PUT_OK:
+ onCommitLogAppend(msg, result, mappedFile);
+ break;
+ case END_OF_FILE:
+ onCommitLogAppend(msg, result, mappedFile);
+ unlockMappedFile = mappedFile;
+ // Create a new file, re-write the message
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0);
+ if (null == mappedFile) {
+ // XXX: warn and notify me
+ log.error("create mapped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, result));
+ }
+ result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
+ if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) {
+ onCommitLogAppend(msg, result, mappedFile);
+ }
+ break;
+ case MESSAGE_SIZE_EXCEEDED:
+ case PROPERTIES_SIZE_EXCEEDED:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
+ case UNKNOWN_ERROR:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
+ default:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
+ }
- result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
- switch (result.getStatus()) {
- case PUT_OK:
- break;
- case END_OF_FILE:
- unlockMappedFile = mappedFile;
- // Create a new file, re-write the message
- mappedFile = this.mappedFileQueue.getLastMappedFile(0);
- if (null == mappedFile) {
- // XXX: warn and notify me
- log.error("create mapped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result));
- }
- result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
- break;
- case MESSAGE_SIZE_EXCEEDED:
- case PROPERTIES_SIZE_EXCEEDED:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
- case UNKNOWN_ERROR:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
- default:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
+ elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
+ beginTimeInLock = 0;
+ } finally {
+ putMessageLock.unlock();
}
-
- elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
} finally {
- beginTimeInLock = 0;
- putMessageLock.unlock();
+ topicQueueLock.unlock(topicQueueKey);
}
if (elapsedTimeInLock > 500) {
@@@ -776,58 -938,90 +952,91 @@@
MappedFile unlockMappedFile = null;
MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
+ long currOffset;
+ if (mappedFile == null) {
+ currOffset = 0;
+ } else {
+ currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();
+ }
+
+ boolean needHandleHA = needHandleHA(messageExtBatch);
+ int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();
+
+ if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableSlaveActingMaster()) {
+ int inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),
+ this.defaultMessageStore.getHaService().inSyncSlaveNums(currOffset) + 1);
+ needAckNums = calcNeedAckNums(inSyncReplicas);
+ if (needAckNums > inSyncReplicas) {
+ // Tell the producer, don't have enough slaves to handle the send request
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));
+ }
+ }
+
//fine-grained lock instead of the coarse-grained
PutMessageThreadLocal pmThreadLocal = this.putMessageThreadLocal.get();
+ updateMaxMessageSize(pmThreadLocal);
MessageExtEncoder batchEncoder = pmThreadLocal.getEncoder();
- PutMessageContext putMessageContext = new PutMessageContext(generateKey(pmThreadLocal.getKeyBuilder(), messageExtBatch));
+ String topicQueueKey = generateKey(pmThreadLocal.getKeyBuilder(), messageExtBatch);
+
+ PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);
messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch, putMessageContext));
- putMessageLock.lock();
+ topicQueueLock.lock(topicQueueKey);
try {
- long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
- this.beginTimeInLock = beginLockTimestamp;
+ defaultMessageStore.assignOffset(messageExtBatch, (short) putMessageContext.getBatchSize());
- // Here settings are stored timestamp, in order to ensure an orderly
- // global
- messageExtBatch.setStoreTimestamp(beginLockTimestamp);
+ putMessageLock.lock();
+ try {
+ long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
+ this.beginTimeInLock = beginLockTimestamp;
- if (null == mappedFile || mappedFile.isFull()) {
- mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
- }
- if (null == mappedFile) {
- log.error("Create mapped file1 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null));
- }
+ // Here settings are stored timestamp, in order to ensure an orderly
+ // global
+ messageExtBatch.setStoreTimestamp(beginLockTimestamp);
- result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);
- switch (result.getStatus()) {
- case PUT_OK:
- break;
- case END_OF_FILE:
- unlockMappedFile = mappedFile;
- // Create a new file, re-write the message
- mappedFile = this.mappedFileQueue.getLastMappedFile(0);
- if (null == mappedFile) {
- // XXX: warn and notify me
- log.error("Create mapped file2 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result));
- }
- result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);
- break;
- case MESSAGE_SIZE_EXCEEDED:
- case PROPERTIES_SIZE_EXCEEDED:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
- case UNKNOWN_ERROR:
- default:
- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
- }
+ if (null == mappedFile || mappedFile.isFull()) {
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
+ }
+ if (null == mappedFile) {
+ log.error("Create mapped file1 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null));
+ }
- elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
+ result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);
+ switch (result.getStatus()) {
+ case PUT_OK:
+ break;
+ case END_OF_FILE:
+ unlockMappedFile = mappedFile;
+ // Create a new file, re-write the message
+ mappedFile = this.mappedFileQueue.getLastMappedFile(0);
+ if (null == mappedFile) {
+ // XXX: warn and notify me
+ log.error("Create mapped file2 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString());
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, result));
+ }
+ result = mappedFile.appendMessages(messageExtBatch, this.appendMessageCallback, putMessageContext);
+ break;
+ case MESSAGE_SIZE_EXCEEDED:
+ case PROPERTIES_SIZE_EXCEEDED:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
+ case UNKNOWN_ERROR:
+ default:
+ beginTimeInLock = 0;
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
+ }
+
+ elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
+ beginTimeInLock = 0;
+ } finally {
+ putMessageLock.unlock();
+ }
} finally {
- beginTimeInLock = 0;
- putMessageLock.unlock();
+ topicQueueLock.unlock(topicQueueKey);
}
if (elapsedTimeInLock > 500) {
@@@ -1527,13 -1869,8 +1865,15 @@@
final int msgLen = calMsgLength(msgInner.getSysFlag(), bodyLength, topicLength, propertiesLength);
+ // Exceeds the maximum message body
+ if (bodyLength > this.maxMessageBodySize) {
+ CommitLog.log.warn("message body size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength
+ + ", maxMessageSize: " + this.maxMessageBodySize);
+ return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+ }
+
+ final long queueOffset = msgInner.getQueueOffset();
+
// Exceeds the maximum message
if (msgLen > this.maxMessageSize) {
CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength
@@@ -1541,52 -1878,49 +1881,52 @@@
return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
}
- // Initialization of storage space
- this.resetByteBuffer(encoderBuffer, msgLen);
// 1 TOTALSIZE
- this.encoderBuffer.putInt(msgLen);
+ this.byteBuf.writeInt(msgLen);
// 2 MAGICCODE
- this.encoderBuffer.putInt(CommitLog.MESSAGE_MAGIC_CODE);
+ this.byteBuf.writeInt(CommitLog.MESSAGE_MAGIC_CODE);
// 3 BODYCRC
- this.encoderBuffer.putInt(msgInner.getBodyCRC());
+ this.byteBuf.writeInt(msgInner.getBodyCRC());
// 4 QUEUEID
- this.encoderBuffer.putInt(msgInner.getQueueId());
+ this.byteBuf.writeInt(msgInner.getQueueId());
// 5 FLAG
- this.encoderBuffer.putInt(msgInner.getFlag());
+ this.byteBuf.writeInt(msgInner.getFlag());
- // 6 QUEUEOFFSET, need update later
- this.byteBuf.writeLong(0);
+ // 6 QUEUEOFFSET
- this.encoderBuffer.putLong(queueOffset);
++ this.byteBuf.writeLong(queueOffset);
// 7 PHYSICALOFFSET, need update later
- this.encoderBuffer.putLong(0);
+ this.byteBuf.writeLong(0);
// 8 SYSFLAG
- this.encoderBuffer.putInt(msgInner.getSysFlag());
+ this.byteBuf.writeInt(msgInner.getSysFlag());
// 9 BORNTIMESTAMP
- this.encoderBuffer.putLong(msgInner.getBornTimestamp());
+ this.byteBuf.writeLong(msgInner.getBornTimestamp());
+
// 10 BORNHOST
- socketAddress2ByteBuffer(msgInner.getBornHost(), this.encoderBuffer);
+ ByteBuffer bornHostBytes = msgInner.getBornHostBytes();
+ this.byteBuf.writeBytes(bornHostBytes.array());
+
// 11 STORETIMESTAMP
- this.encoderBuffer.putLong(msgInner.getStoreTimestamp());
+ this.byteBuf.writeLong(msgInner.getStoreTimestamp());
+
// 12 STOREHOSTADDRESS
- socketAddress2ByteBuffer(msgInner.getStoreHost(), this.encoderBuffer);
+ ByteBuffer storeHostBytes = msgInner.getStoreHostBytes();
+ this.byteBuf.writeBytes(storeHostBytes.array());
+
// 13 RECONSUMETIMES
- this.encoderBuffer.putInt(msgInner.getReconsumeTimes());
+ this.byteBuf.writeInt(msgInner.getReconsumeTimes());
// 14 Prepared Transaction Offset
- this.encoderBuffer.putLong(msgInner.getPreparedTransactionOffset());
+ this.byteBuf.writeLong(msgInner.getPreparedTransactionOffset());
// 15 BODY
- this.encoderBuffer.putInt(bodyLength);
+ this.byteBuf.writeInt(bodyLength);
if (bodyLength > 0)
- this.encoderBuffer.put(msgInner.getBody());
+ this.byteBuf.writeBytes(msgInner.getBody());
// 16 TOPIC
- this.encoderBuffer.put((byte) topicLength);
- this.encoderBuffer.put(topicData);
+ this.byteBuf.writeByte((byte) topicLength);
+ this.byteBuf.writeBytes(topicData);
// 17 PROPERTIES
- this.encoderBuffer.putShort((short) propertiesLength);
+ this.byteBuf.writeShort((short) propertiesLength);
if (propertiesLength > 0)
- this.encoderBuffer.put(propertiesData);
+ this.byteBuf.writeBytes(propertiesData);
- encoderBuffer.flip();
return null;
}
@@@ -1639,54 -1973,63 +1979,54 @@@
final int topicLength = topicData.length;
int totalPropLen = needAppendLastPropertySeparator ? propertiesLen + batchPropLen + 1
- : propertiesLen + batchPropLen;
+ : propertiesLen + batchPropLen;
final int msgLen = calMsgLength(messageExtBatch.getSysFlag(), bodyLen, topicLength, totalPropLen);
- // Exceeds the maximum message
- if (msgLen > this.maxMessageSize) {
- CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLen
- + ", maxMessageSize: " + this.maxMessageSize);
- throw new RuntimeException("message size exceeded");
- }
-
- totalMsgLen += msgLen;
- // Determines whether there is sufficient free space
- if (totalMsgLen > maxMessageSize) {
- throw new RuntimeException("message size exceeded");
- }
-
// 1 TOTALSIZE
- this.encoderBuffer.putInt(msgLen);
+ this.byteBuf.writeInt(msgLen);
// 2 MAGICCODE
- this.encoderBuffer.putInt(CommitLog.MESSAGE_MAGIC_CODE);
+ this.byteBuf.writeInt(CommitLog.MESSAGE_MAGIC_CODE);
// 3 BODYCRC
- this.encoderBuffer.putInt(bodyCrc);
+ this.byteBuf.writeInt(bodyCrc);
// 4 QUEUEID
- this.encoderBuffer.putInt(messageExtBatch.getQueueId());
+ this.byteBuf.writeInt(messageExtBatch.getQueueId());
// 5 FLAG
- this.encoderBuffer.putInt(flag);
+ this.byteBuf.writeInt(flag);
// 6 QUEUEOFFSET
- this.encoderBuffer.putLong(0);
+ this.byteBuf.writeLong(0);
// 7 PHYSICALOFFSET
- this.encoderBuffer.putLong(0);
+ this.byteBuf.writeLong(0);
// 8 SYSFLAG
- this.encoderBuffer.putInt(messageExtBatch.getSysFlag());
+ this.byteBuf.writeInt(messageExtBatch.getSysFlag());
// 9 BORNTIMESTAMP
- this.encoderBuffer.putLong(messageExtBatch.getBornTimestamp());
+ this.byteBuf.writeLong(messageExtBatch.getBornTimestamp());
+
// 10 BORNHOST
- this.resetByteBuffer(bornHostHolder, bornHostLength);
- this.encoderBuffer.put(messageExtBatch.getBornHostBytes(bornHostHolder));
+ ByteBuffer bornHostBytes = messageExtBatch.getBornHostBytes();
+ this.byteBuf.writeBytes(bornHostBytes.array());
+
// 11 STORETIMESTAMP
- this.encoderBuffer.putLong(messageExtBatch.getStoreTimestamp());
+ this.byteBuf.writeLong(messageExtBatch.getStoreTimestamp());
+
// 12 STOREHOSTADDRESS
- this.resetByteBuffer(storeHostHolder, storeHostLength);
- this.encoderBuffer.put(messageExtBatch.getStoreHostBytes(storeHostHolder));
+ ByteBuffer storeHostBytes = messageExtBatch.getStoreHostBytes();
+ this.byteBuf.writeBytes(storeHostBytes.array());
+
// 13 RECONSUMETIMES
- this.encoderBuffer.putInt(messageExtBatch.getReconsumeTimes());
+ this.byteBuf.writeInt(messageExtBatch.getReconsumeTimes());
// 14 Prepared Transaction Offset, batch does not support transaction
- this.encoderBuffer.putLong(0);
+ this.byteBuf.writeLong(0);
// 15 BODY
- this.encoderBuffer.putInt(bodyLen);
+ this.byteBuf.writeInt(bodyLen);
if (bodyLen > 0)
- this.encoderBuffer.put(messagesByteBuff.array(), bodyPos, bodyLen);
+ this.byteBuf.writeBytes(messagesByteBuff.array(), bodyPos, bodyLen);
// 16 TOPIC
- this.encoderBuffer.put((byte) topicLength);
- this.encoderBuffer.put(topicData);
+ this.byteBuf.writeByte((byte) topicLength);
+ this.byteBuf.writeBytes(topicData);
// 17 PROPERTIES
- this.encoderBuffer.putShort((short) totalPropLen);
+ this.byteBuf.writeShort((short) totalPropLen);
if (propertiesLen > 0) {
- this.encoderBuffer.put(messagesByteBuff.array(), propertiesPos, propertiesLen);
+ this.byteBuf.writeBytes(messagesByteBuff.array(), propertiesPos, propertiesLen);
}
if (batchPropLen > 0) {
if (needAppendLastPropertySeparator) {
@@@ -1697,72 -2040,174 +2037,184 @@@
}
putMessageContext.setBatchSize(batchSize);
putMessageContext.setPhyPos(new long[batchSize]);
- encoderBuffer.flip();
- return encoderBuffer;
+
+ return this.byteBuf.nioBuffer();
}
- private void resetByteBuffer(final ByteBuffer byteBuffer, final int limit) {
- byteBuffer.flip();
- byteBuffer.limit(limit);
+ public ByteBuffer getEncoderBuffer() {
+ return this.byteBuf.nioBuffer();
}
+ public int getMaxMessageBodySize() {
+ return this.maxMessageBodySize;
+ }
+
+ public void updateEncoderBufferCapacity(int newMaxMessageBodySize) {
+ this.maxMessageBodySize = newMaxMessageBodySize;
+ //Reserve 64kb for encoding buffer outside body
+ this.maxMessageSize = Integer.MAX_VALUE - newMaxMessageBodySize >= 64 * 1024 ?
+ this.maxMessageBodySize + 64 * 1024 : Integer.MAX_VALUE;
+ this.byteBuf.capacity(this.maxMessageSize);
+ }
}
- static class PutMessageThreadLocal {
- private MessageExtEncoder encoder;
- private StringBuilder keyBuilder;
+ interface FlushManager {
+ void start();
- PutMessageThreadLocal(int maxMessageBodySize) {
- encoder = new MessageExtEncoder(maxMessageBodySize);
- keyBuilder = new StringBuilder();
- }
+ void shutdown();
- public MessageExtEncoder getEncoder() {
- return encoder;
+ void wakeUpFlush();
+
+ void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt);
+
+ CompletableFuture<PutMessageStatus> handleDiskFlush(AppendMessageResult result, MessageExt messageExt);
+ }
+
+ class DefaultFlushManager implements FlushManager {
+
+ private final FlushCommitLogService flushCommitLogService;
+
+ //If TransientStorePool enabled, we must flush message to FileChannel at fixed periods
+ private final FlushCommitLogService commitLogService;
+
+ public DefaultFlushManager() {
+ if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
+ this.flushCommitLogService = new CommitLog.GroupCommitService();
+ } else {
+ this.flushCommitLogService = new CommitLog.FlushRealTimeService();
+ }
+
+ this.commitLogService = new CommitLog.CommitRealTimeService();
}
- public StringBuilder getKeyBuilder() {
- return keyBuilder;
+ @Override
+ public void start() {
+ this.flushCommitLogService.start();
+
+ if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
+ this.commitLogService.start();
+ }
}
- }
- static class PutMessageContext {
- private String topicQueueTableKey;
- private long[] phyPos;
- private int batchSize;
+ public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult,
+ MessageExt messageExt) {
+ // Synchronization flush
+ if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
+ final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
+ if (messageExt.isWaitStoreMsgOK()) {
+ GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(), CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
+ service.putRequest(request);
+ CompletableFuture<PutMessageStatus> flushOkFuture = request.future();
+ PutMessageStatus flushStatus = null;
+ try {
+ flushStatus = flushOkFuture.get(CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout(),
+ TimeUnit.MILLISECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ //flushOK=false;
+ }
+ if (flushStatus != PutMessageStatus.PUT_OK) {
+ log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()
+ + " client address: " + messageExt.getBornHostString());
+ putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);
+ }
+ } else {
+ service.wakeup();
+ }
+ }
+ // Asynchronous flush
+ else {
+ if (!CommitLog.this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
+ flushCommitLogService.wakeup();
+ } else {
+ commitLogService.wakeup();
+ }
+ }
+ }
- public PutMessageContext(String topicQueueTableKey) {
- this.topicQueueTableKey = topicQueueTableKey;
+ @Override
+ public CompletableFuture<PutMessageStatus> handleDiskFlush(AppendMessageResult result, MessageExt messageExt) {
+ // Synchronization flush
+ if (FlushDiskType.SYNC_FLUSH == CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
+ final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
+ if (messageExt.isWaitStoreMsgOK()) {
+ GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(), CommitLog.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
+ flushDiskWatcher.add(request);
+ service.putRequest(request);
+ return request.future();
+ } else {
+ service.wakeup();
+ return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);
+ }
+ }
+ // Asynchronous flush
+ else {
+ if (!CommitLog.this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
+ flushCommitLogService.wakeup();
+ } else {
+ commitLogService.wakeup();
+ }
+ return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);
+ }
}
- public String getTopicQueueTableKey() {
- return topicQueueTableKey;
+ @Override
+ public void wakeUpFlush() {
+ // now wake up flush thread.
+ flushCommitLogService.wakeup();
}
- public long[] getPhyPos() {
- return phyPos;
+ @Override
+ public void shutdown() {
+ if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
+ this.commitLogService.shutdown();
+ }
+
+ this.flushCommitLogService.shutdown();
}
- public void setPhyPos(long[] phyPos) {
- this.phyPos = phyPos;
+ }
+
+ public int getCommitLogSize() {
+ return commitLogSize;
+ }
+
+ public MappedFileQueue getMappedFileQueue() {
+ return mappedFileQueue;
+ }
+
+ public MessageStore getMessageStore() {
+ return defaultMessageStore;
+ }
+
+ @Override
+ public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) {
+ this.getMappedFileQueue().swapMap(reserveNum, forceSwapIntervalMs, normalSwapIntervalMs);
+ }
+
+ public boolean isMappedFilesEmpty() {
+ return this.mappedFileQueue.isMappedFilesEmpty();
+ }
+
+ @Override
+ public void cleanSwappedMap(long forceCleanSwapIntervalMs) {
+ this.getMappedFileQueue().cleanSwappedMap(forceCleanSwapIntervalMs);
+ }
+
+ static class PutMessageThreadLocal {
+ private MessageExtEncoder encoder;
+ private StringBuilder keyBuilder;
+
- PutMessageThreadLocal(int size) {
- encoder = new MessageExtEncoder(size);
++ PutMessageThreadLocal(int maxMessageBodySize) {
++ encoder = new MessageExtEncoder(maxMessageBodySize);
+ keyBuilder = new StringBuilder();
}
- public int getBatchSize() {
- return batchSize;
+ public MessageExtEncoder getEncoder() {
+ return encoder;
}
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
+ public StringBuilder getKeyBuilder() {
+ return keyBuilder;
}
}
}
diff --cc store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
index 8cbe690ea,0da12a232..1cb60b87d
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
@@@ -16,7 -16,7 +16,8 @@@
*/
package org.apache.rocketmq.store;
+ import com.google.common.hash.Hashing;
+import io.openmessaging.storage.dledger.entry.DLedgerEntry;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
@@@ -305,6 -346,60 +347,60 @@@ public class DefaultMessageStore implem
this.shutdown = false;
}
+ private void doRecheckReputOffsetFromCq() throws InterruptedException {
+ if (!messageStoreConfig.isRecheckReputOffsetFromCq()) {
+ return;
+ }
+
+ /**
+ * 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog;
+ * 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go;
+ * 3. Calculate the reput offset according to the consume queue;
+ * 4. Make sure the fall-behind messages to be dispatched before starting the commitlog, especially when the broker role are automatically changed.
+ */
+ long maxPhysicalPosInLogicQueue = commitLog.getMinOffset();
+ for (ConcurrentMap<Integer, ConsumeQueueInterface> maps : this.getConsumeQueueTable().values()) {
+ for (ConsumeQueueInterface logic : maps.values()) {
+ if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) {
+ maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset();
+ }
+ }
+ }
+ // If maxPhyPos(CQs) < minPhyPos(CommitLog), some newly deleted topics may be re-dispatched into cqs mistakenly.
+ if (maxPhysicalPosInLogicQueue < 0) {
+ maxPhysicalPosInLogicQueue = 0;
+ }
+ if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) {
+ maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset();
+ /**
+ * This happens in following conditions:
+ * 1. If someone removes all the consumequeue files or the disk get damaged.
+ * 2. Launch a new broker, and copy the commitlog from other brokers.
+ *
+ * All the conditions has the same in common that the maxPhysicalPosInLogicQueue should be 0.
+ * If the maxPhysicalPosInLogicQueue is gt 0, there maybe something wrong.
+ */
+ LOGGER.warn("[TooSmallCqOffset] maxPhysicalPosInLogicQueue={} clMinOffset={}", maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset());
+ }
+ LOGGER.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMinOffset={} clMaxOffset={} clConfirmedOffset={}",
- maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset());
++ maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset());
+ this.reputMessageService.setReputFromOffset(maxPhysicalPosInLogicQueue);
+
+ /**
+ * 1. Finish dispatching the messages fall behind, then to start other services.
+ * 2. DLedger committedPos may be missing, so here just require dispatchBehindBytes <= 0
+ */
+ while (true) {
+ if (dispatchBehindBytes() <= 0) {
+ break;
+ }
+ Thread.sleep(1000);
+ LOGGER.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes());
+ }
+ this.recoverTopicQueueTable();
+ }
+
+ @Override
public void shutdown() {
if (!this.shutdown) {
this.shutdown = true;
@@@ -446,21 -481,27 +482,27 @@@
@Override
public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {
- PutMessageStatus checkStoreStatus = this.checkStoreStatus();
- if (checkStoreStatus != PutMessageStatus.PUT_OK) {
- return CompletableFuture.completedFuture(new PutMessageResult(checkStoreStatus, null));
- }
- PutMessageStatus msgCheckStatus = this.checkMessage(msg);
- if (msgCheckStatus == PutMessageStatus.MESSAGE_ILLEGAL) {
- return CompletableFuture.completedFuture(new PutMessageResult(msgCheckStatus, null));
+ for (PutMessageHook putMessageHook : putMessageHookList) {
+ PutMessageResult handleResult = putMessageHook.executeBeforePutMessage(msg);
+ if (handleResult != null) {
+ return CompletableFuture.completedFuture(handleResult);
+ }
}
- PutMessageStatus lmqMsgCheckStatus = this.checkLmqMessage(msg);
- if (msgCheckStatus == PutMessageStatus.LMQ_CONSUME_QUEUE_NUM_EXCEEDED) {
- return CompletableFuture.completedFuture(new PutMessageResult(lmqMsgCheckStatus, null));
+ if (msg.getProperties().containsKey(MessageConst.PROPERTY_INNER_NUM)
- && !MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
++ && !MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
+ LOGGER.warn("[BUG]The message had property {} but is not an inner batch", MessageConst.PROPERTY_INNER_NUM);
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));
}
+ if (MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
+ Optional<TopicConfig> topicConfig = this.getTopicConfig(msg.getTopic());
+ if (!QueueTypeUtils.isBatchCq(topicConfig)) {
+ LOGGER.error("[BUG]The message is an inner batch but cq type is not batch cq");
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));
+ }
+ }
long beginTime = this.getSystemClock().now();
CompletableFuture<PutMessageResult> putResultFuture = this.commitLog.asyncPutMessage(msg);
@@@ -522,15 -563,15 +564,15 @@@
private PutMessageResult waitForPutResult(CompletableFuture<PutMessageResult> putMessageResultFuture) {
try {
int putMessageTimeout =
-- Math.max(this.messageStoreConfig.getSyncFlushTimeout(),
-- this.messageStoreConfig.getSlaveTimeout()) + 5000;
++ Math.max(this.messageStoreConfig.getSyncFlushTimeout(),
++ this.messageStoreConfig.getSlaveTimeout()) + 5000;
return putMessageResultFuture.get(putMessageTimeout, TimeUnit.MILLISECONDS);
} catch (ExecutionException | InterruptedException e) {
return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);
} catch (TimeoutException e) {
- log.error("usually it will never timeout, putMessageTimeout is much bigger than slaveTimeout and "
- + "flushTimeout so the result can be got anyway, but in some situations timeout will happen like full gc "
- + "process hangs or other unexpected situations.");
+ LOGGER.error("usually it will never timeout, putMessageTimeout is much bigger than slaveTimeout and "
- + "flushTimeout so the result can be got anyway, but in some situations timeout will happen like full gc "
- + "process hangs or other unexpected situations.");
++ + "flushTimeout so the result can be got anyway, but in some situations timeout will happen like full gc "
++ + "process hangs or other unexpected situations.");
return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null);
}
}
@@@ -995,15 -1142,29 +1144,15 @@@
lastQueryMsgTime = msg.getStoreTimestamp();
}
-// String[] keyArray = msg.getKeys().split(MessageConst.KEY_SEPARATOR);
-// if (topic.equals(msg.getTopic())) {
-// for (String k : keyArray) {
-// if (k.equals(key)) {
-// match = true;
-// break;
-// }
-// }
-// }
-
- if (match) {
- SelectMappedBufferResult result = this.commitLog.getData(offset, false);
- if (result != null) {
- int size = result.getByteBuffer().getInt(0);
- result.getByteBuffer().limit(size);
- result.setSize(size);
- queryMessageResult.addMessage(result);
- }
- } else {
- LOGGER.warn("queryMessage hash duplicate, topic={}, key={}", topic, key);
+ SelectMappedBufferResult result = this.commitLog.getData(offset, false);
+ if (result != null) {
+ int size = result.getByteBuffer().getInt(0);
+ result.getByteBuffer().limit(size);
+ result.setSize(size);
+ queryMessageResult.addMessage(result);
}
} catch (Exception e) {
- log.error("queryMessage exception", e);
+ LOGGER.error("queryMessage exception", e);
}
}
@@@ -1139,9 -1293,10 +1281,10 @@@
int msgIdLength = (inetSocketAddress.getAddress() instanceof Inet6Address) ? 16 + 4 + 8 : 4 + 4 + 8;
final ByteBuffer msgIdMemory = ByteBuffer.allocate(msgIdLength);
String msgId =
- MessageDecoder.createMessageId(msgIdMemory, MessageExt.socketAddress2ByteBuffer(storeHost), offsetPy);
+ MessageDecoder.createMessageId(msgIdMemory, MessageExt.socketAddress2ByteBuffer(storeHost), offsetPy);
- messageIds.put(msgId, nextOffset++);
- if (nextOffset > maxOffset) {
+ messageIds.put(msgId, cqUnit.getQueueOffset());
+ nextOffset = cqUnit.getQueueOffset() + cqUnit.getBatchNum();
+ if (nextOffset >= maxOffset) {
return messageIds;
}
}
diff --cc store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
index 46797b343,1a6d6ae48..5f1ee6989
--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
+++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
@@@ -85,9 -89,11 +93,11 @@@ public class MessageStoreConfig
// The number of hours to keep a log file before deleting it (in hours)
@ImportantField
private int fileReservedTime = 72;
+ @ImportantField
+ private int deleteFileBatchMax = 10;
// Flow control for ConsumeQueue
private int putMsgIndexHightWater = 600000;
- // The maximum size of message,default is 4M
+ // The maximum size of message body,default is 4M,4M only for body length,not include others.
private int maxMessageSize = 1024 * 1024 * 4;
// Whether check the CRC32 of the records consumed.
// This ensures no on-the-wire or on-disk corruption to the messages occurred.
diff --cc store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
index 5474fa46b,42064a35a..ac7d31fa3
--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
@@@ -55,8 -50,7 +50,8 @@@ import org.apache.rocketmq.store.PutMes
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.StoreStatsService;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
- import org.apache.rocketmq.store.schedule.ScheduleMessageService;
+ import org.apache.rocketmq.store.logfile.MappedFile;
/**
* Store all metadata downtime for recovery, data protection reliability
diff --cc store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAConnection.java
index 5d88f00f2,3b7241065..aa5da1eb4
--- a/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAConnection.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAConnection.java
@@@ -26,24 -27,24 +27,25 @@@ import org.apache.rocketmq.common.const
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.apache.rocketmq.remoting.netty.NettySystemConfig;
import org.apache.rocketmq.store.SelectMappedBufferResult;
- public class HAConnection {
+ public class DefaultHAConnection implements HAConnection {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
- private final HAService haService;
+ private final DefaultHAService haService;
private final SocketChannel socketChannel;
- private final String clientAddr;
- private final WriteSocketService writeSocketService;
- private final ReadSocketService readSocketService;
-
+ private final String clientAddress;
+ private WriteSocketService writeSocketService;
+ private ReadSocketService readSocketService;
+ private volatile HAConnectionState currentState = HAConnectionState.TRANSFER;
private volatile long slaveRequestOffset = -1;
private volatile long slaveAckOffset = -1;
+ private FlowMonitor flowMonitor;
- public HAConnection(final HAService haService, final SocketChannel socketChannel) throws IOException {
+ public DefaultHAConnection(final DefaultHAService haService, final SocketChannel socketChannel) throws IOException {
this.haService = haService;
this.socketChannel = socketChannel;
- this.clientAddr = this.socketChannel.socket().getRemoteSocketAddress().toString();
+ this.clientAddress = this.socketChannel.socket().getRemoteSocketAddress().toString();
this.socketChannel.configureBlocking(false);
this.socketChannel.socket().setSoLinger(false, -1);
this.socketChannel.socket().setTcpNoDelay(true);
@@@ -382,6 -431,9 +436,9 @@@
@Override
public String getServiceName() {
+ if (haService.getDefaultMessageStore().getBrokerConfig().isInBrokerContainer()) {
- return haService.getDefaultMessageStore().getBrokerConfig().getLoggerIdentifier() + WriteSocketService.class.getSimpleName();
++ return haService.getDefaultMessageStore().getBrokerIdentity().getLoggerIdentifier() + WriteSocketService.class.getSimpleName();
+ }
return WriteSocketService.class.getSimpleName();
}
diff --cc store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
index bdea3307b,8b59f547e..1684eed12
--- a/store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
+++ b/store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
@@@ -19,7 -19,10 +19,8 @@@ package org.apache.rocketmq.store.index
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
import java.util.List;
+ import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
@@@ -32,15 -36,18 +34,16 @@@ public class IndexFile
private static int invalidIndex = 0;
private final int hashSlotNum;
private final int indexNum;
+ private final int fileTotalSize;
private final MappedFile mappedFile;
- private final FileChannel fileChannel;
private final MappedByteBuffer mappedByteBuffer;
private final IndexHeader indexHeader;
public IndexFile(final String fileName, final int hashSlotNum, final int indexNum,
final long endPhyOffset, final long endTimestamp) throws IOException {
- int fileTotalSize =
+ this.fileTotalSize =
IndexHeader.INDEX_HEADER_SIZE + (hashSlotNum * hashSlotSize) + (indexNum * indexSize);
- this.mappedFile = new MappedFile(fileName, fileTotalSize);
+ this.mappedFile = new DefaultMappedFile(fileName, fileTotalSize);
- this.fileChannel = this.mappedFile.getFileChannel();
this.mappedByteBuffer = this.mappedFile.getMappedByteBuffer();
this.hashSlotNum = hashSlotNum;
this.indexNum = indexNum;
@@@ -67,6 -78,17 +74,11 @@@
this.indexHeader.load();
}
+ public void shutdown() {
+ this.flush();
+ UtilAll.cleanBuffer(this.mappedByteBuffer);
-
- try {
- this.fileChannel.close();
- } catch (IOException e) {
- log.error("Shutdown error in index file", e);
- }
+ }
+
public void flush() {
long beginTime = System.currentTimeMillis();
if (this.mappedFile.hold()) {
diff --cc store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
index b1fcbf20b,095bb261f..89709989f
--- a/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
+++ b/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
@@@ -377,12 -376,13 +376,13 @@@ public class DefaultMappedFile extends
}
if (commitLeastPages > 0) {
- return ((write / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE)) >= commitLeastPages;
+ return ((write / OS_PAGE_SIZE) - (commit / OS_PAGE_SIZE)) >= commitLeastPages;
}
- return write > flush;
+ return write > commit;
}
+ @Override
public int getFlushedPosition() {
return flushedPosition.get();
}
diff --cc store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
index 605552bd0,1ea987ada..7ec9477d7
--- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
+++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
@@@ -19,9 -19,19 +19,20 @@@ package org.apache.rocketmq.store.stats
import java.util.HashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
+ import org.apache.commons.lang3.tuple.Pair;
+ import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.ThreadFactoryImpl;
+ import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
+ import org.apache.rocketmq.common.statistics.StatisticsItem;
+ import org.apache.rocketmq.common.statistics.StatisticsItemFormatter;
+ import org.apache.rocketmq.common.statistics.StatisticsItemPrinter;
+ import org.apache.rocketmq.common.statistics.StatisticsItemScheduledIncrementPrinter;
+ import org.apache.rocketmq.common.statistics.StatisticsItemScheduledPrinter;
+ import org.apache.rocketmq.common.statistics.StatisticsItemStateGetter;
+ import org.apache.rocketmq.common.statistics.StatisticsKindMeta;
+ import org.apache.rocketmq.common.statistics.StatisticsManager;
+import org.apache.rocketmq.common.stats.Stats;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.stats.MomentStatsItemSet;
@@@ -30,38 -40,81 +41,85 @@@ import org.apache.rocketmq.common.stats
public class BrokerStatsManager {
- public static final String QUEUE_PUT_NUMS = "QUEUE_PUT_NUMS";
- public static final String QUEUE_PUT_SIZE = "QUEUE_PUT_SIZE";
- public static final String QUEUE_GET_NUMS = "QUEUE_GET_NUMS";
- public static final String QUEUE_GET_SIZE = "QUEUE_GET_SIZE";
- public static final String TOPIC_PUT_NUMS = "TOPIC_PUT_NUMS";
- public static final String TOPIC_PUT_SIZE = "TOPIC_PUT_SIZE";
+ @Deprecated public static final String QUEUE_PUT_NUMS = Stats.QUEUE_PUT_NUMS;
+ @Deprecated public static final String QUEUE_PUT_SIZE = Stats.QUEUE_PUT_SIZE;
+ @Deprecated public static final String QUEUE_GET_NUMS = Stats.QUEUE_GET_NUMS;
+ @Deprecated public static final String QUEUE_GET_SIZE = Stats.QUEUE_GET_SIZE;
+ @Deprecated public static final String TOPIC_PUT_NUMS = Stats.TOPIC_PUT_NUMS;
+ @Deprecated public static final String TOPIC_PUT_SIZE = Stats.TOPIC_PUT_SIZE;
++
+ @Deprecated public static final String GROUP_GET_NUMS = Stats.GROUP_GET_NUMS;
+ @Deprecated public static final String GROUP_GET_SIZE = Stats.GROUP_GET_SIZE;
++
+ @Deprecated public static final String SNDBCK_PUT_NUMS = Stats.SNDBCK_PUT_NUMS;
+ @Deprecated public static final String BROKER_PUT_NUMS = Stats.BROKER_PUT_NUMS;
+ @Deprecated public static final String BROKER_GET_NUMS = Stats.BROKER_GET_NUMS;
+ @Deprecated public static final String GROUP_GET_FROM_DISK_NUMS = Stats.GROUP_GET_FROM_DISK_NUMS;
+ @Deprecated public static final String GROUP_GET_FROM_DISK_SIZE = Stats.GROUP_GET_FROM_DISK_SIZE;
+ @Deprecated public static final String BROKER_GET_FROM_DISK_NUMS = Stats.BROKER_GET_FROM_DISK_NUMS;
+ @Deprecated public static final String BROKER_GET_FROM_DISK_SIZE = Stats.BROKER_GET_FROM_DISK_SIZE;
+ // For commercial
+ @Deprecated public static final String COMMERCIAL_SEND_TIMES = Stats.COMMERCIAL_SEND_TIMES;
+ @Deprecated public static final String COMMERCIAL_SNDBCK_TIMES = Stats.COMMERCIAL_SNDBCK_TIMES;
+ @Deprecated public static final String COMMERCIAL_RCV_TIMES = Stats.COMMERCIAL_RCV_TIMES;
+ @Deprecated public static final String COMMERCIAL_RCV_EPOLLS = Stats.COMMERCIAL_RCV_EPOLLS;
+ @Deprecated public static final String COMMERCIAL_SEND_SIZE = Stats.COMMERCIAL_SEND_SIZE;
+ @Deprecated public static final String COMMERCIAL_RCV_SIZE = Stats.COMMERCIAL_RCV_SIZE;
+ @Deprecated public static final String COMMERCIAL_PERM_FAILURES = Stats.COMMERCIAL_PERM_FAILURES;
++
++
+ // Send message latency
+ public static final String TOPIC_PUT_LATENCY = "TOPIC_PUT_LATENCY";
- public static final String GROUP_GET_NUMS = "GROUP_GET_NUMS";
- public static final String GROUP_GET_SIZE = "GROUP_GET_SIZE";
+ public static final String GROUP_ACK_NUMS = "GROUP_ACK_NUMS";
+ public static final String GROUP_CK_NUMS = "GROUP_CK_NUMS";
- public static final String SNDBCK_PUT_NUMS = "SNDBCK_PUT_NUMS";
+ public static final String DLQ_PUT_NUMS = "DLQ_PUT_NUMS";
- public static final String BROKER_PUT_NUMS = "BROKER_PUT_NUMS";
- public static final String BROKER_GET_NUMS = "BROKER_GET_NUMS";
+ public static final String BROKER_ACK_NUMS = "BROKER_ACK_NUMS";
+ public static final String BROKER_CK_NUMS = "BROKER_CK_NUMS";
- public static final String GROUP_GET_FROM_DISK_NUMS = "GROUP_GET_FROM_DISK_NUMS";
- public static final String GROUP_GET_FROM_DISK_SIZE = "GROUP_GET_FROM_DISK_SIZE";
- public static final String BROKER_GET_FROM_DISK_NUMS = "BROKER_GET_FROM_DISK_NUMS";
- public static final String BROKER_GET_FROM_DISK_SIZE = "BROKER_GET_FROM_DISK_SIZE";
-
+ public static final String SNDBCK2DLQ_TIMES = "SNDBCK2DLQ_TIMES";
+
- // For commercial
- public static final String COMMERCIAL_SEND_TIMES = "COMMERCIAL_SEND_TIMES";
- public static final String COMMERCIAL_SNDBCK_TIMES = "COMMERCIAL_SNDBCK_TIMES";
- public static final String COMMERCIAL_RCV_TIMES = "COMMERCIAL_RCV_TIMES";
- public static final String COMMERCIAL_RCV_EPOLLS = "COMMERCIAL_RCV_EPOLLS";
- public static final String COMMERCIAL_SEND_SIZE = "COMMERCIAL_SEND_SIZE";
- public static final String COMMERCIAL_RCV_SIZE = "COMMERCIAL_RCV_SIZE";
- public static final String COMMERCIAL_PERM_FAILURES = "COMMERCIAL_PERM_FAILURES";
public static final String COMMERCIAL_OWNER = "Owner";
- // Message Size limit for one api-calling count.
- public static final double SIZE_PER_COUNT = 64 * 1024;
+
+ public static final String ACCOUNT_OWNER_PARENT = "OWNER_PARENT";
+ public static final String ACCOUNT_OWNER_SELF = "OWNER_SELF";
+
+ public static final long ACCOUNT_STAT_INVERTAL = 60 * 1000;
+ public static final String ACCOUNT_AUTH_TYPE = "AUTH_TYPE";
+
+ public static final String ACCOUNT_SEND = "SEND";
+ public static final String ACCOUNT_RCV = "RCV";
+ public static final String ACCOUNT_SEND_BACK = "SEND_BACK";
+ public static final String ACCOUNT_SEND_BACK_TO_DLQ = "SEND_BACK_TO_DLQ";
+ public static final String ACCOUNT_AUTH_FAILED = "AUTH_FAILED";
+ public static final String ACCOUNT_SEND_REJ = "SEND_REJ";
+ public static final String ACCOUNT_REV_REJ = "RCV_REJ";
+
+ public static final String MSG_NUM = "MSG_NUM";
+ public static final String MSG_SIZE = "MSG_SIZE";
+ public static final String SUCCESS_MSG_NUM = "SUCCESS_MSG_NUM";
+ public static final String FAILURE_MSG_NUM = "FAILURE_MSG_NUM";
+ public static final String COMMERCIAL_MSG_NUM = "COMMERCIAL_MSG_NUM";
+ public static final String SUCCESS_REQ_NUM = "SUCCESS_REQ_NUM";
+ public static final String FAILURE_REQ_NUM = "FAILURE_REQ_NUM";
+ public static final String SUCCESS_MSG_SIZE = "SUCCESS_MSG_SIZE";
+ public static final String FAILURE_MSG_SIZE = "FAILURE_MSG_SIZE";
+ public static final String RT = "RT";
+ public static final String INNER_RT = "INNER_RT";
- public static final String GROUP_GET_FALL_SIZE = "GROUP_GET_FALL_SIZE";
- public static final String GROUP_GET_FALL_TIME = "GROUP_GET_FALL_TIME";
+ @Deprecated public static final String GROUP_GET_FALL_SIZE = Stats.GROUP_GET_FALL_SIZE;
+ @Deprecated public static final String GROUP_GET_FALL_TIME = Stats.GROUP_GET_FALL_TIME;
// Pull Message Latency
- public static final String GROUP_GET_LATENCY = "GROUP_GET_LATENCY";
+ @Deprecated public static final String GROUP_GET_LATENCY = Stats.GROUP_GET_LATENCY;
+
+ // Consumer Register Time
+ public static final String CONSUMER_REGISTER_TIME = "CONSUMER_REGISTER_TIME";
+ // Producer Register Time
+ public static final String PRODUCER_REGISTER_TIME = "PRODUCER_REGISTER_TIME";
+ public static final String CHANNEL_ACTIVITY = "CHANNEL_ACTIVITY";
+ public static final String CHANNEL_ACTIVITY_CONNECT = "CONNECT";
+ public static final String CHANNEL_ACTIVITY_IDLE = "IDLE";
+ public static final String CHANNEL_ACTIVITY_EXCEPTION = "EXCEPTION";
+ public static final String CHANNEL_ACTIVITY_CLOSE = "CLOSE";
+
/**
* read disk follow stats
*/
@@@ -80,33 -151,130 +156,130 @@@
public BrokerStatsManager(String clusterName, boolean enableQueueStat) {
this.clusterName = clusterName;
this.enableQueueStat = enableQueueStat;
+ initScheduleService();
+ init();
+ }
+
+ public void init() {
+ momentStatsItemSetFallSize = new MomentStatsItemSet(GROUP_GET_FALL_SIZE,
+ scheduledExecutorService, log);
+
+ momentStatsItemSetFallTime = new MomentStatsItemSet(GROUP_GET_FALL_TIME,
+ scheduledExecutorService, log);
if (enableQueueStat) {
- this.statsTable.put(QUEUE_PUT_NUMS, new StatsItemSet(QUEUE_PUT_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(QUEUE_PUT_SIZE, new StatsItemSet(QUEUE_PUT_SIZE, this.scheduledExecutorService, log));
- this.statsTable.put(QUEUE_GET_NUMS, new StatsItemSet(QUEUE_GET_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(QUEUE_GET_SIZE, new StatsItemSet(QUEUE_GET_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.QUEUE_PUT_NUMS, new StatsItemSet(Stats.QUEUE_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.QUEUE_PUT_SIZE, new StatsItemSet(Stats.QUEUE_PUT_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.QUEUE_GET_NUMS, new StatsItemSet(Stats.QUEUE_GET_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.QUEUE_GET_SIZE, new StatsItemSet(Stats.QUEUE_GET_SIZE, this.scheduledExecutorService, log));
}
- this.statsTable.put(TOPIC_PUT_NUMS, new StatsItemSet(TOPIC_PUT_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(TOPIC_PUT_SIZE, new StatsItemSet(TOPIC_PUT_SIZE, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_NUMS, new StatsItemSet(GROUP_GET_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_SIZE, new StatsItemSet(GROUP_GET_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.TOPIC_PUT_NUMS, new StatsItemSet(Stats.TOPIC_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.TOPIC_PUT_SIZE, new StatsItemSet(Stats.TOPIC_PUT_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.GROUP_GET_NUMS, new StatsItemSet(Stats.GROUP_GET_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.GROUP_GET_SIZE, new StatsItemSet(Stats.GROUP_GET_SIZE, this.scheduledExecutorService, log));
+ this.statsTable.put(GROUP_ACK_NUMS, new StatsItemSet(GROUP_ACK_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(GROUP_CK_NUMS, new StatsItemSet(GROUP_CK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_LATENCY, new StatsItemSet(GROUP_GET_LATENCY, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.GROUP_GET_LATENCY, new StatsItemSet(Stats.GROUP_GET_LATENCY, this.scheduledExecutorService, log));
+ this.statsTable.put(TOPIC_PUT_LATENCY, new StatsItemSet(TOPIC_PUT_LATENCY, this.scheduledExecutorService, log));
- this.statsTable.put(SNDBCK_PUT_NUMS, new StatsItemSet(SNDBCK_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.SNDBCK_PUT_NUMS, new StatsItemSet(Stats.SNDBCK_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(DLQ_PUT_NUMS, new StatsItemSet(DLQ_PUT_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(BROKER_PUT_NUMS, new StatsItemSet(BROKER_PUT_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(BROKER_GET_NUMS, new StatsItemSet(BROKER_GET_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.BROKER_PUT_NUMS, new StatsItemSet(Stats.BROKER_PUT_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(Stats.BROKER_GET_NUMS, new StatsItemSet(Stats.BROKER_GET_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(Stats.GROUP_GET_FROM_DISK_NUMS, new StatsItemSet(Stats.GROUP_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(Stats.GROUP_GET_FROM_DISK_SIZE, new StatsItemSet(Stats.GROUP_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
- this.statsTable.put(Stats.BROKER_GET_FROM_DISK_NUMS, new StatsItemSet(Stats.BROKER_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(Stats.BROKER_GET_FROM_DISK_SIZE, new StatsItemSet(Stats.BROKER_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
-
- this.statsTable.put(Stats.COMMERCIAL_SEND_TIMES, new StatsItemSet(Stats.COMMERCIAL_SEND_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_RCV_TIMES, new StatsItemSet(Stats.COMMERCIAL_RCV_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_SEND_SIZE, new StatsItemSet(Stats.COMMERCIAL_SEND_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_RCV_SIZE, new StatsItemSet(Stats.COMMERCIAL_RCV_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_RCV_EPOLLS, new StatsItemSet(Stats.COMMERCIAL_RCV_EPOLLS, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_SNDBCK_TIMES, new StatsItemSet(Stats.COMMERCIAL_SNDBCK_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(Stats.COMMERCIAL_PERM_FAILURES, new StatsItemSet(Stats.COMMERCIAL_PERM_FAILURES, this.commercialExecutor, COMMERCIAL_LOG));
+ this.statsTable.put(BROKER_ACK_NUMS, new StatsItemSet(BROKER_ACK_NUMS, this.scheduledExecutorService, log));
+ this.statsTable.put(BROKER_CK_NUMS, new StatsItemSet(BROKER_CK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_FROM_DISK_NUMS,
- new StatsItemSet(GROUP_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(GROUP_GET_FROM_DISK_SIZE,
- new StatsItemSet(GROUP_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
- this.statsTable.put(BROKER_GET_FROM_DISK_NUMS,
- new StatsItemSet(BROKER_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
- this.statsTable.put(BROKER_GET_FROM_DISK_SIZE,
- new StatsItemSet(BROKER_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
++ this.statsTable.put(Stats.GROUP_GET_FROM_DISK_NUMS,
++ new StatsItemSet(Stats.GROUP_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
++ this.statsTable.put(Stats.GROUP_GET_FROM_DISK_SIZE,
++ new StatsItemSet(Stats.GROUP_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
++ this.statsTable.put(Stats.BROKER_GET_FROM_DISK_NUMS,
++ new StatsItemSet(Stats.BROKER_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log));
++ this.statsTable.put(Stats.BROKER_GET_FROM_DISK_SIZE,
++ new StatsItemSet(Stats.BROKER_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log));
+
+ this.statsTable.put(SNDBCK2DLQ_TIMES,
+ new StatsItemSet(SNDBCK2DLQ_TIMES, this.scheduledExecutorService, DLQ_STAT_LOG));
+
- this.statsTable.put(COMMERCIAL_SEND_TIMES,
- new StatsItemSet(COMMERCIAL_SEND_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_RCV_TIMES,
- new StatsItemSet(COMMERCIAL_RCV_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_SEND_SIZE,
- new StatsItemSet(COMMERCIAL_SEND_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_RCV_SIZE,
- new StatsItemSet(COMMERCIAL_RCV_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_RCV_EPOLLS,
- new StatsItemSet(COMMERCIAL_RCV_EPOLLS, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_SNDBCK_TIMES,
- new StatsItemSet(COMMERCIAL_SNDBCK_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
- this.statsTable.put(COMMERCIAL_PERM_FAILURES,
- new StatsItemSet(COMMERCIAL_PERM_FAILURES, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_SEND_TIMES,
++ new StatsItemSet(Stats.COMMERCIAL_SEND_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_RCV_TIMES,
++ new StatsItemSet(Stats.COMMERCIAL_RCV_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_SEND_SIZE,
++ new StatsItemSet(Stats.COMMERCIAL_SEND_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_RCV_SIZE,
++ new StatsItemSet(Stats.COMMERCIAL_RCV_SIZE, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_RCV_EPOLLS,
++ new StatsItemSet(Stats.COMMERCIAL_RCV_EPOLLS, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_SNDBCK_TIMES,
++ new StatsItemSet(Stats.COMMERCIAL_SNDBCK_TIMES, this.commercialExecutor, COMMERCIAL_LOG));
++ this.statsTable.put(Stats.COMMERCIAL_PERM_FAILURES,
++ new StatsItemSet(Stats.COMMERCIAL_PERM_FAILURES, this.commercialExecutor, COMMERCIAL_LOG));
+
+ this.statsTable.put(CONSUMER_REGISTER_TIME,
+ new StatsItemSet(CONSUMER_REGISTER_TIME, this.scheduledExecutorService, log));
+ this.statsTable.put(PRODUCER_REGISTER_TIME,
+ new StatsItemSet(PRODUCER_REGISTER_TIME, this.scheduledExecutorService, log));
+
+ this.statsTable.put(CHANNEL_ACTIVITY, new StatsItemSet(CHANNEL_ACTIVITY, this.scheduledExecutorService, log));
+
+ StatisticsItemFormatter formatter = new StatisticsItemFormatter();
+ accountStatManager.setBriefMeta(new Pair[] {
+ Pair.of(RT, new long[][] {{50, 50}, {100, 10}, {1000, 10}}),
+ Pair.of(INNER_RT, new long[][] {{10, 10}, {100, 10}, {1000, 10}})});
+ String[] itemNames = new String[] {
+ MSG_NUM, SUCCESS_MSG_NUM, FAILURE_MSG_NUM, COMMERCIAL_MSG_NUM,
+ SUCCESS_REQ_NUM, FAILURE_REQ_NUM,
+ MSG_SIZE, SUCCESS_MSG_SIZE, FAILURE_MSG_SIZE,
+ RT, INNER_RT};
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_SEND, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_RCV, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_SEND_BACK, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_SEND_BACK_TO_DLQ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_SEND_REJ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.addStatisticsKindMeta(createStatisticsKindMeta(
+ ACCOUNT_REV_REJ, itemNames, this.accountExecutor, formatter, ACCOUNT_LOG, ACCOUNT_STAT_INVERTAL));
+ this.accountStatManager.setStatisticsItemStateGetter(new StatisticsItemStateGetter() {
+ @Override
+ public boolean online(StatisticsItem item) {
+ String[] strArr = null;
+ try {
+ strArr = splitAccountStatKey(item.getStatObject());
+ } catch (Exception e) {
+ log.warn("parse account stat key failed, key: {}", item.getStatObject());
+ return false;
+ }
+
+ // TODO ugly
+ if (strArr == null || strArr.length < 4) {
+ return false;
+ }
+
+ String instanceId = strArr[1];
+ String topic = strArr[2];
+ String group = strArr[3];
+
+ String kind = item.getStatKind();
+ if (ACCOUNT_SEND.equals(kind) || ACCOUNT_SEND_REJ.equals(kind)) {
+ return produerStateGetter.online(instanceId, group, topic);
+ } else if (ACCOUNT_RCV.equals(kind) || ACCOUNT_SEND_BACK.equals(kind) || ACCOUNT_SEND_BACK_TO_DLQ.equals(kind) || ACCOUNT_REV_REJ.equals(kind)) {
+ return consumerStateGetter.online(instanceId, group, topic);
+ }
+ return false;
+ }
+ });
+ }
+
+ private void initScheduleService() {
+ this.scheduledExecutorService =
+ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig));
+ this.commercialExecutor =
+ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig));
+ this.accountExecutor =
+ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig));
}
public MomentStatsItemSet getMomentStatsItemSetFallSize() {
@@@ -196,8 -380,32 +385,32 @@@
}
}
+ public void incConsumerRegisterTime(final int incValue) {
+ this.statsTable.get(CONSUMER_REGISTER_TIME).addValue(this.clusterName, incValue, 1);
+ }
+
+ public void incProducerRegisterTime(final int incValue) {
+ this.statsTable.get(PRODUCER_REGISTER_TIME).addValue(this.clusterName, incValue, 1);
+ }
+
+ public void incChannelConnectNum() {
+ this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_CONNECT, 1, 1);
+ }
+
+ public void incChannelCloseNum() {
+ this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_CLOSE, 1, 1);
+ }
+
+ public void incChannelExceptionNum() {
+ this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_EXCEPTION, 1, 1);
+ }
+
+ public void incChannelIdleNum() {
+ this.statsTable.get(CHANNEL_ACTIVITY).addValue(CHANNEL_ACTIVITY_IDLE, 1, 1);
+ }
+
public void incTopicPutNums(final String topic) {
- this.statsTable.get(TOPIC_PUT_NUMS).addValue(topic, 1, 1);
+ this.statsTable.get(Stats.TOPIC_PUT_NUMS).addValue(topic, 1, 1);
}
public void incTopicPutNums(final String topic, int num, int times) {
@@@ -210,9 -418,19 +423,19 @@@
public void incGroupGetNums(final String group, final String topic, final int incValue) {
final String statsKey = buildStatsKey(topic, group);
- this.statsTable.get(GROUP_GET_NUMS).addValue(statsKey, incValue, 1);
+ this.statsTable.get(Stats.GROUP_GET_NUMS).addValue(statsKey, incValue, 1);
}
+ public void incGroupCkNums(final String group, final String topic, final int incValue) {
+ final String statsKey = buildStatsKey(topic, group);
+ this.statsTable.get(GROUP_CK_NUMS).addValue(statsKey, incValue, 1);
+ }
+
+ public void incGroupAckNums(final String group, final String topic, final int incValue) {
+ final String statsKey = buildStatsKey(topic, group);
+ this.statsTable.get(GROUP_ACK_NUMS).addValue(statsKey, incValue, 1);
+ }
+
public String buildStatsKey(String topic, String group) {
StringBuilder strBuilder;
if (topic != null && group != null) {
@@@ -269,11 -487,16 +492,16 @@@
} else {
statsKey = buildStatsKey(topic, group);
}
- this.statsTable.get(GROUP_GET_LATENCY).addRTValue(statsKey, incValue, 1);
+ this.statsTable.get(Stats.GROUP_GET_LATENCY).addRTValue(statsKey, incValue, 1);
}
+ public void incTopicPutLatency(final String topic, final int queueId, final int incValue) {
+ final String statsKey = String.format("%d@%s", queueId, topic);
+ this.statsTable.get(TOPIC_PUT_LATENCY).addValue(statsKey, incValue, 1);
+ }
+
public void incBrokerPutNums() {
- this.statsTable.get(BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(1);
+ this.statsTable.get(Stats.BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(1);
}
public void incBrokerPutNums(final int incValue) {
@@@ -281,12 -504,20 +509,20 @@@
}
public void incBrokerGetNums(final int incValue) {
- this.statsTable.get(BROKER_GET_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
+ this.statsTable.get(Stats.BROKER_GET_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
}
+ public void incBrokerAckNums(final int incValue) {
+ this.statsTable.get(BROKER_ACK_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
+ }
+
+ public void incBrokerCkNums(final int incValue) {
+ this.statsTable.get(BROKER_CK_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
+ }
+
public void incSendBackNums(final String group, final String topic) {
final String statsKey = buildStatsKey(topic, group);
- this.statsTable.get(SNDBCK_PUT_NUMS).addValue(statsKey, 1, 1);
+ this.statsTable.get(Stats.SNDBCK_PUT_NUMS).addValue(statsKey, 1, 1);
}
public double tpsGroupGetNums(final String group, final String topic) {
diff --cc store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java
index 1b57b2d20,d299a47b3..9dc8a8b24
--- a/store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java
@@@ -56,9 -56,9 +56,9 @@@ public class AppendCallbackTest
messageStoreConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "unitteststore");
messageStoreConfig.setStorePathCommitLog(System.getProperty("user.home") + File.separator + "unitteststore" + File.separator + "commitlog");
//too much reference
- DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, null, null, null);
+ DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, null, null, new BrokerConfig());
CommitLog commitLog = new CommitLog(messageStore);
- callback = commitLog.new DefaultAppendMessageCallback(1024);
+ callback = commitLog.new DefaultAppendMessageCallback();
}
@After
diff --cc store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java
index 690ad0cbe,a9cd6c68c..6d42d6236
--- a/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java
@@@ -187,51 -185,26 +186,33 @@@ public class BatchPutMessageTest
}
}
+ private String generateKey(StringBuilder keyBuilder, MessageExt messageExt) {
+ keyBuilder.setLength(0);
+ keyBuilder.append(messageExt.getTopic());
+ keyBuilder.append('-');
+ keyBuilder.append(messageExt.getQueueId());
+ return keyBuilder.toString();
+ }
- @Test
- public void testPutLongBatchMessage() throws Exception{
- String topic = "batch-long-topic";
- MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) messageStore).getMessageStoreConfig();
- CommitLog commitLog = ((DefaultMessageStore) messageStore).getCommitLog();
- CommitLog.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get();
-
- MessageExtBatch messageExtBatch = new MessageExtBatch();
- messageExtBatch.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]);
- messageExtBatch.setTopic(topic);
- CommitLog.PutMessageContext putMessageContext = new CommitLog.PutMessageContext(generateKey(
- putMessageThreadLocal.getKeyBuilder(), messageExtBatch));
- RuntimeException runtimeException = assertThrows(RuntimeException.class,
- () -> putMessageThreadLocal.getEncoder().encode(messageExtBatch, putMessageContext));
- assertThat("message body size exceeded").isEqualTo(runtimeException.getMessage());
- }
-
-
private int calMsgLength(int bodyLength, int topicLength, int propertiesLength) {
final int msgLen = 4 //TOTALSIZE
- + 4 //MAGICCODE
- + 4 //BODYCRC
- + 4 //QUEUEID
- + 4 //FLAG
- + 8 //QUEUEOFFSET
- + 8 //PHYSICALOFFSET
- + 4 //SYSFLAG
- + 8 //BORNTIMESTAMP
- + 8 //BORNHOST
- + 8 //STORETIMESTAMP
- + 8 //STOREHOSTADDRESS
- + 4 //RECONSUMETIMES
- + 8 //Prepared Transaction Offset
- + 4 + (bodyLength > 0 ? bodyLength : 0) //BODY
- + 1 + topicLength //TOPIC
- + 2 + (propertiesLength > 0 ? propertiesLength : 0) //propertiesLength
- + 0;
+ + 4 //MAGICCODE
+ + 4 //BODYCRC
+ + 4 //QUEUEID
+ + 4 //FLAG
+ + 8 //QUEUEOFFSET
+ + 8 //PHYSICALOFFSET
+ + 4 //SYSFLAG
+ + 8 //BORNTIMESTAMP
+ + 8 //BORNHOST
+ + 8 //STORETIMESTAMP
+ + 8 //STOREHOSTADDRESS
+ + 4 //RECONSUMETIMES
+ + 8 //Prepared Transaction Offset
+ + 4 + (bodyLength > 0 ? bodyLength : 0) //BODY
+ + 1 + topicLength //TOPIC
+ + 2 + (propertiesLength > 0 ? propertiesLength : 0) //propertiesLength
+ + 0;
return msgLen;
}
diff --cc store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
index 491437d80,b39b9bb03..db9b3c665
--- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
@@@ -40,6 -46,9 +46,8 @@@ import org.apache.rocketmq.store.config
import org.apache.rocketmq.store.config.FlushDiskType;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
-import org.apache.rocketmq.store.logfile.DefaultMappedFile;
+ import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+ import org.apache.rocketmq.store.queue.CqUnit;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
import org.junit.After;
import org.junit.Before;
@@@ -644,80 -709,91 +708,151 @@@ public class DefaultMessageStoreTest
}
@Test
- public void testCleanUnusedLmqTopic() throws Exception {
- String lmqTopic = "%LMQ%123";
+ public void testPutMsgExceedsMaxLength() {
+ messageBody = new byte[4 * 1024 * 1024 + 1];
+ MessageExtBrokerInner msg = buildMessage();
- MessageExtBrokerInner messageExtBrokerInner = buildMessage();
- messageExtBrokerInner.setTopic("test");
- messageExtBrokerInner.setQueueId(0);
- messageExtBrokerInner.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqTopic);
- messageStore.putMessage(messageExtBrokerInner);
+ PutMessageResult result = messageStore.putMessage(msg);
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL);
+ }
+
+ @Test
+ public void testPutMsgBatchExceedsMaxLength() {
+ messageBody = new byte[4 * 1024 * 1024 + 1];
+ MessageExtBrokerInner msg1 = buildMessage();
+ MessageExtBrokerInner msg2 = buildMessage();
+ MessageExtBrokerInner msg3 = buildMessage();
+
+ MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3));
+ msgBatch.setBody(msgBatch.encode());
+
+ MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch);
+
+ try {
+ PutMessageResult result = this.messageStore.putMessages(msgExtBatch);
+ } catch (Exception e) {
- assertThat(e.getMessage()).contains("message size exceeded");
++ assertThat(e.getMessage()).contains("message body size exceeded");
+ }
+ }
+
+ @Test
+ public void testPutMsgWhenReplicasNotEnough() {
+ MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) this.messageStore).getMessageStoreConfig();
+ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
+ messageStoreConfig.setTotalReplicas(2);
+ messageStoreConfig.setInSyncReplicas(2);
+ messageStoreConfig.setEnableAutoInSyncReplicas(false);
+ ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);
+ this.messageStore.setAliveReplicaNumInGroup(1);
- Thread.sleep(3000);
- messageStore.cleanUnusedLmqTopic(lmqTopic);
+ MessageExtBrokerInner msg = buildMessage();
+ PutMessageResult result = this.messageStore.putMessage(msg);
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH);
+ ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);
+ }
+
+
+ @Test
+ public void testPutMsgWhenAdaptiveDegradation () {
+ MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) this.messageStore).getMessageStoreConfig();
+ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
+ messageStoreConfig.setTotalReplicas(2);
+ messageStoreConfig.setInSyncReplicas(2);
+ messageStoreConfig.setEnableAutoInSyncReplicas(true);
+ ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true);
+ this.messageStore.setAliveReplicaNumInGroup(1);
+
+ MessageExtBrokerInner msg = buildMessage();
+ PutMessageResult result = this.messageStore.putMessage(msg);
+ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);
+ ((DefaultMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false);
+ messageStoreConfig.setEnableAutoInSyncReplicas(false);
+ }
+
+
+ @Test
+ public void testGetBulkCommitLogData() {
+ DefaultMessageStore defaultMessageStore = (DefaultMessageStore) messageStore;
+
+ messageBody = new byte[2 * 1024 * 1024];
+
+ for (int i = 0; i < 10; i++) {
+ MessageExtBrokerInner msg1 = buildMessage();
+ messageStore.putMessage(msg1);
+ }
+
+ System.out.printf("%d%n", defaultMessageStore.getMaxPhyOffset());
+
+ List<SelectMappedBufferResult> bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset());
+ List<MessageExt> msgList = new ArrayList<>();
+ for (SelectMappedBufferResult bufferResult : bufferResultList) {
+ msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false));
+ bufferResult.release();
+ }
+ assertThat(msgList.size()).isEqualTo(10);
}
+ @Test
+ public void testPutLongMessage() throws Exception{
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
+ CommitLog commitLog = ((DefaultMessageStore) messageStore).getCommitLog();
+ MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) messageStore).getMessageStoreConfig();
+ CommitLog.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get();
+
+ //body size, topic size, properties size exactly equal to max size
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
+ messageExtBrokerInner.setTopic(new String(new byte[127]));
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));
+ PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult1 == null);
+
+ //body size exactly more than max message body size
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]);
+ PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult2.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+
+ //body size exactly equal to max message size
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]);
+ PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult3.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+
+ //message properties length more than properties maxSize
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE+1]));
+ PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult4.getPutMessageStatus() == PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);
+
+ //message length more than buffer length capacity
+ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]);
+ messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE]));
+ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE]));
+ PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner);
+ assertTrue(encodeResult5.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+ }
+
+ @Test
+ public void testDynamicMaxMessageSize(){
+ MessageExtBrokerInner messageExtBrokerInner = buildMessage();
+ MessageStoreConfig messageStoreConfig = ((DefaultMessageStore) messageStore).getMessageStoreConfig();
+ int originMaxMessageSize = messageStoreConfig.getMaxMessageSize();
+
+ messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]);
+ PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner);
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+
+ int newMaxMessageSize = originMaxMessageSize + 10;
+ messageStoreConfig.setMaxMessageSize(newMaxMessageSize);
+ putMessageResult = messageStore.putMessage(messageExtBrokerInner);
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK);
+
+ messageStoreConfig.setMaxMessageSize(10);
+ putMessageResult = messageStore.putMessage(messageExtBrokerInner);
+ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL);
+
+ messageStoreConfig.setMaxMessageSize(originMaxMessageSize);
+ }
+
private class MyMessageArrivingListener implements MessageArrivingListener {
@Override
public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime,
diff --cc store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
index cdf3f1610,627f9dd2a..a7e726c57
--- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
@@@ -27,8 -31,9 +31,8 @@@ import org.junit.After
import org.junit.Before;
import org.junit.Test;
+ import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue;
import static org.junit.Assert.assertEquals;
--import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
diff --cc store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java
index 5660de136,7a2e32a21..9dc724cd3
--- a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java
+++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java
@@@ -20,6 -20,8 +20,7 @@@ import org.apache.rocketmq.common.UtilA
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExtBatch;
+ import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-import org.apache.rocketmq.store.logfile.DefaultMappedFile;
import org.junit.After;
import java.io.File;
diff --cc test/pom.xml
index 0a6f5edb4,3cf4f2d61..c3b2db596
--- a/test/pom.xml
+++ b/test/pom.xml
@@@ -40,10 -40,10 +40,14 @@@
<groupId>${project.groupId}</groupId>
<artifactId>rocketmq-namesrv</artifactId>
</dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>rocketmq-container</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>annotations-api</artifactId>
+ </dependency>
<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
diff --cc test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
index 6a13049f7,6a13049f7..a186e73fe
--- a/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
+++ b/test/src/main/java/org/apache/rocketmq/test/listener/AbstractListener.java
@@@ -29,7 -29,7 +29,7 @@@ import org.apache.rocketmq.test.util.Te
public class AbstractListener extends MQCollector implements MessageListener {
public static Logger logger = Logger.getLogger(AbstractListener.class);
-- protected boolean isDebug = false;
++ protected boolean isDebug = true;
protected String listenerName = null;
protected Collection<Object> allSendMsgs = null;
diff --cc test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
index 000000000,1d8098053..d50ff718f
mode 000000,100644..100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
@@@ -1,0 -1,97 +1,96 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.rocketmq.test.client.consumer.pop;
+
+ import org.apache.rocketmq.common.message.MessageClientExt;
+ import org.apache.rocketmq.common.message.MessageConst;
+ import org.apache.rocketmq.common.message.MessageRequestMode;
+ import org.apache.rocketmq.logging.inner.Logger;
+ import org.apache.rocketmq.test.base.BaseConf;
+ import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
+ import org.apache.rocketmq.test.client.rmq.RMQPopConsumer;
+ import org.apache.rocketmq.test.factory.ConsumerFactory;
+ import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
+ import org.apache.rocketmq.test.util.RandomUtil;
+ import org.apache.rocketmq.test.util.TestUtils;
+ import org.apache.rocketmq.test.util.VerifyUtils;
+ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+ import org.junit.After;
+ import org.junit.Assert;
+ import org.junit.Before;
-import org.junit.Ignore;
+ import org.junit.Test;
+
+ import static com.google.common.truth.Truth.assertThat;
+
+ public class PopSubCheckIT extends BaseConf {
+ private static Logger logger = Logger.getLogger(PopSubCheckIT.class);
+ private String group;
+
+ private DefaultMQAdminExt defaultMQAdminExt;
+
+ @Before
+ public void setUp() throws Exception {
+ group = initConsumerGroup();
+
+ defaultMQAdminExt = new DefaultMQAdminExt();
+ defaultMQAdminExt.setInstanceName(RandomUtil.getStringByUUID());
+ defaultMQAdminExt.start();
+ }
+
+ @After
+ public void tearDown() {
+ defaultMQAdminExt.shutdown();
+ super.shutdown();
+ }
+
- @Ignore
++
+ @Test
+ public void testNormalPopAck() throws Exception {
+ String topic = initTopic();
+ logger.info(String.format("use topic: %s; group: %s !", topic, group));
+
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ producer.getProducer().setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);
+
+ for (String brokerAddr : new String[]{brokerController1.getBrokerAddr(), brokerController2.getBrokerAddr()}) {
+ defaultMQAdminExt.setMessageRequestMode(brokerAddr, topic, group, MessageRequestMode.POP, 8, 60_000);
+ }
+
+ RMQPopConsumer consumer = ConsumerFactory.getRMQPopConsumer(nsAddr, group,
+ topic, "*", new RMQNormalListener());
+ mqClients.add(consumer);
+
+ int msgNum = 1;
+ producer.send(msgNum);
+ Assert.assertEquals("Not all sent succeeded", msgNum, producer.getAllUndupMsgBody().size());
+ logger.info(producer.getFirstMsg());
+
+ TestUtils.waitForSeconds(10);
+
+ consumer.getListener().waitForMessageConsume(msgNum, 30_000);
+ assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(), consumer.getListener().getAllMsgBody()))
+ .containsExactlyElementsIn(producer.getAllMsgBody());
+ for (Object o : consumer.getListener().getAllOriginMsg()) {
+ MessageClientExt msg = (MessageClientExt) o;
+ assertThat(msg.getProperty(MessageConst.PROPERTY_POP_CK)).named("check pop meta").isNotEmpty();
+ }
+
+ consumer.getListener().waitForMessageConsume(msgNum, 3_000 * 9);
+ assertThat(consumer.getListener().getAllOriginMsg().size()).isEqualTo(msgNum);
+ }
+ }
diff --cc test/src/test/java/org/apache/rocketmq/test/container/ContainerIntegrationTestBase.java
index 000000000,4a2e06642..e1cadaf45
mode 000000,100644..100644
--- a/test/src/test/java/org/apache/rocketmq/test/container/ContainerIntegrationTestBase.java
+++ b/test/src/test/java/org/apache/rocketmq/test/container/ContainerIntegrationTestBase.java
@@@ -1,0 -1,666 +1,666 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.rocketmq.test.container;
+
+ import ch.qos.logback.classic.LoggerContext;
+ import ch.qos.logback.classic.joran.JoranConfigurator;
+ import io.netty.channel.ChannelHandlerContext;
+ import java.io.File;
+ import java.io.IOException;
+ import java.nio.file.Files;
+ import java.time.Duration;
+ import java.util.ArrayList;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Random;
+ import java.util.Set;
+ import java.util.UUID;
+ import java.util.concurrent.ConcurrentHashMap;
+ import java.util.concurrent.ConcurrentMap;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicInteger;
+ import org.apache.commons.lang3.builder.EqualsBuilder;
+ import org.apache.commons.lang3.builder.HashCodeBuilder;
+ import org.apache.rocketmq.container.BrokerContainer;
+ import org.apache.rocketmq.container.InnerSalveBrokerController;
+ import org.apache.rocketmq.broker.BrokerController;
+ import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+ import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+ import org.apache.rocketmq.client.producer.DefaultMQProducer;
+ import org.apache.rocketmq.client.producer.TransactionCheckListener;
+ import org.apache.rocketmq.client.producer.TransactionMQProducer;
+ import org.apache.rocketmq.common.BrokerConfig;
+ import org.apache.rocketmq.container.BrokerContainerConfig;
+ import org.apache.rocketmq.common.BrokerIdentity;
+ import org.apache.rocketmq.common.MQVersion;
+ import org.apache.rocketmq.common.TopicConfig;
+ import org.apache.rocketmq.common.UtilAll;
+ import org.apache.rocketmq.common.message.MessageQueue;
+ import org.apache.rocketmq.common.namesrv.NamesrvConfig;
+ import org.apache.rocketmq.common.protocol.RequestCode;
+ import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader;
+ import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+ import org.apache.rocketmq.logging.InternalLogger;
+ import org.apache.rocketmq.logging.InternalLoggerFactory;
+ import org.apache.rocketmq.namesrv.NamesrvController;
+ import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+ import org.apache.rocketmq.store.config.BrokerRole;
+ import org.apache.rocketmq.store.config.MessageStoreConfig;
+ import org.apache.rocketmq.store.ha.HAConnection;
+ import org.apache.rocketmq.store.ha.HAConnectionState;
+ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+ import org.junit.Assert;
+ import org.junit.BeforeClass;
+ import org.slf4j.LoggerFactory;
+
+ import static org.awaitility.Awaitility.await;
+
+ /**
+ * ContainerIntegrationTestBase will setup a rocketmq ha cluster contains two broker group:
+ * <li>BrokerA contains two replicas</li>
+ * <li>BrokerB contains three replicas</li>
+ */
+ public class ContainerIntegrationTestBase {
+ private static final AtomicBoolean CLUSTER_SET_UP = new AtomicBoolean(false);
+ private static final List<File> TMP_FILE_LIST = new ArrayList<>();
+ private static final Random RANDOM = new Random();
+ protected static String nsAddr;
+
+ protected static final String THREE_REPLICAS_TOPIC = "SEND_MESSAGE_TEST_TOPIC_THREE_REPLICAS";
+
+ protected static final List<BrokerContainer> brokerContainerList = new ArrayList<>();
+ protected static final List<NamesrvController> namesrvControllers = new ArrayList<>();
+
+ protected static final String BROKER_NAME_PREFIX = "TestBrokerName_";
+ protected static final int COMMIT_LOG_SIZE = 128 * 1024;
+ protected static final int INDEX_NUM = 1000;
+ protected static final AtomicInteger BROKER_INDEX = new AtomicInteger(0);
+
+ protected static BrokerContainer brokerContainer1;
+ protected static BrokerContainer brokerContainer2;
+ protected static BrokerContainer brokerContainer3;
+ protected static BrokerController master1With3Replicas;
+ protected static BrokerController master2With3Replicas;
+ protected static BrokerController master3With3Replicas;
+ protected static NamesrvController namesrvController;
+
+ protected static DefaultMQAdminExt defaultMQAdminExt;
+
+ private final static InternalLogger LOG = InternalLoggerFactory.getLogger(ContainerIntegrationTestBase.class);
+ private static ConcurrentMap<BrokerConfig, MessageStoreConfig> slaveStoreConfigCache = new ConcurrentHashMap<>();
+
+ protected static ConcurrentMap<BrokerConfigLite, BrokerController> isolatedBrokers = new ConcurrentHashMap<>();
+ private static final Set<Integer> PORTS_IN_USE = new HashSet<>();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (CLUSTER_SET_UP.compareAndSet(false, true)) {
+ System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));
+ System.setProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "0.99");
+ System.setProperty("rocketmq.broker.diskSpaceWarningLevelRatio", "0.99");
+
+ LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
+ JoranConfigurator configurator = new JoranConfigurator();
+ configurator.setContext(lc);
+ lc.reset();
+ //https://logback.qos.ch/manual/configuration.html
+ lc.setPackagingDataEnabled(false);
+
+ configurator.doConfigure("../distribution/conf/logback_broker.xml");
+ configurator.doConfigure("../distribution/conf/logback_namesrv.xml");
+
+ setUpCluster();
+ setUpTopic();
+ registerCleaner();
+
+ System.out.printf("cluster setup complete%n");
+ }
+ }
+
+ private static void setUpTopic() {
+ createTopic(THREE_REPLICAS_TOPIC);
+ }
+
+ private static void createTopic(String topic) {
+ createTopicTo(master1With3Replicas, topic);
+ createTopicTo(master2With3Replicas, topic);
+ createTopicTo(master3With3Replicas, topic);
+ }
+
+ private static void setUpCluster() throws Exception {
+ namesrvController = createAndStartNamesrv();
+ nsAddr = "127.0.0.1:" + namesrvController.getNettyServerConfig().getListenPort();
+ System.out.printf("namesrv addr: %s%n", nsAddr);
+
+ /*
+ * BrokerContainer1 | BrokerContainer2 | BrokerContainer3
+ *
+ * master1With3Replicas(m) master2With3Replicas(m) master3With3Replicas(m)
+ * master3With3Replicas(s0) master1With3Replicas(s0) master2With3Replicas(s0)
+ * master2With3Replicas(s1) master3With3Replicas(s1) master1With3Replicas(s1)
+ */
+
+ brokerContainer1 = createAndStartBrokerContainer(nsAddr);
+ brokerContainer2 = createAndStartBrokerContainer(nsAddr);
+ brokerContainer3 = createAndStartBrokerContainer(nsAddr);
+ // Create three broker groups, two contains two replicas, another contains three replicas
+ master1With3Replicas = createAndAddMaster(brokerContainer1, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());
+ master2With3Replicas = createAndAddMaster(brokerContainer2, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());
+ master3With3Replicas = createAndAddMaster(brokerContainer3, new BrokerGroupConfig(), BROKER_INDEX.getAndIncrement());
+
+ createAndAddSlave(1, brokerContainer1, master3With3Replicas);
+ createAndAddSlave(1, brokerContainer2, master1With3Replicas);
+ createAndAddSlave(1, brokerContainer3, master2With3Replicas);
+
+ createAndAddSlave(2, brokerContainer1, master2With3Replicas);
+ createAndAddSlave(2, brokerContainer2, master3With3Replicas);
+ createAndAddSlave(2, brokerContainer3, master1With3Replicas);
+
+ awaitUntilSlaveOK();
+
+ defaultMQAdminExt = new DefaultMQAdminExt("HATest_Admin_Group");
+ defaultMQAdminExt.setNamesrvAddr(nsAddr);
+ defaultMQAdminExt.start();
+ }
+
+ protected static void createTopicTo(BrokerController masterBroker, String topicName, int rqn, int wqn) {
+ try {
+ TopicConfig topicConfig = new TopicConfig(topicName, rqn, wqn, 6, 0);
+ defaultMQAdminExt.createAndUpdateTopicConfig(masterBroker.getBrokerAddr(), topicConfig);
+
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer1);
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer2);
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer3);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException("Create topic to broker failed", e);
+ }
+ }
+
+ protected static void createGroup(BrokerController masterBroker, String groupName) {
+ try {
+ SubscriptionGroupConfig config = new SubscriptionGroupConfig();
+ config.setGroupName(groupName);
+
+ masterBroker.getSubscriptionGroupManager().updateSubscriptionGroupConfig(config);
+
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer1);
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer2);
+ triggerSlaveSync(masterBroker.getBrokerConfig().getBrokerName(), brokerContainer3);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException("Create group to broker failed", e);
+ }
+ }
+
+ private static void triggerSlaveSync(String brokerName, BrokerContainer brokerContainer) {
+ for (InnerSalveBrokerController slaveBroker : brokerContainer.getSlaveBrokers()) {
+ if (slaveBroker.getBrokerConfig().getBrokerName().equals(brokerName)) {
+ slaveBroker.getSlaveSynchronize().syncAll();
+ slaveBroker.registerBrokerAll(true, false, true);
+ }
+ }
+ }
+
+ protected static void createTopicTo(BrokerController brokerController, String topicName) {
+ createTopicTo(brokerController, topicName, 8, 8);
+ }
+
+ private static void registerCleaner() {
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ if (CLUSTER_SET_UP.compareAndSet(true, false)) {
+ System.out.printf("clean up%n");
+ defaultMQAdminExt.shutdown();
+
+ for (final BrokerContainer brokerContainer : brokerContainerList) {
+ brokerContainer.shutdown();
+ for (BrokerController brokerController : brokerContainer.getBrokerControllers()) {
+ brokerController.getMessageStore().destroy();
+ }
+ }
+
+ for (final NamesrvController namesrvController : namesrvControllers) {
+ namesrvController.shutdown();
+ }
+
+ for (final File file : TMP_FILE_LIST) {
+ UtilAll.deleteFile(file);
+ }
+ }
+ }));
+ }
+
+ private static File createBaseDir(String prefix) {
+ final File file;
+ try {
+ file = Files.createTempDirectory(prefix).toFile();
+ TMP_FILE_LIST.add(file);
+ System.out.printf("create file at %s%n", file.getAbsolutePath());
+ return file;
+ } catch (IOException e) {
+ throw new RuntimeException("Couldn't create tmp folder", e);
+ }
+ }
+
+ public static NamesrvController createAndStartNamesrv() {
+ String baseDir = createBaseDir("test-cluster-namesrv").getAbsolutePath();
+ NamesrvConfig namesrvConfig = new NamesrvConfig();
+ NettyServerConfig nameServerNettyServerConfig = new NettyServerConfig();
+ namesrvConfig.setKvConfigPath(baseDir + File.separator + "namesrv" + File.separator + "kvConfig.json");
+ namesrvConfig.setConfigStorePath(baseDir + File.separator + "namesrv" + File.separator + "namesrv.properties");
+ namesrvConfig.setSupportActingMaster(true);
+ namesrvConfig.setScanNotActiveBrokerInterval(1000);
+
+ nameServerNettyServerConfig.setListenPort(generatePort(10000, 10000));
+ NamesrvController namesrvController = new NamesrvController(namesrvConfig, nameServerNettyServerConfig);
+ try {
+ Assert.assertTrue(namesrvController.initialize());
+ LOG.info("Name Server Start:{}", nameServerNettyServerConfig.getListenPort());
+ namesrvController.start();
+ } catch (Exception e) {
+ LOG.info("Name Server start failed");
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ namesrvController.getRemotingServer().registerProcessor(RequestCode.REGISTER_BROKER, new NettyRequestProcessor() {
+ @Override
+ public RemotingCommand processRequest(final ChannelHandlerContext ctx,
+ final RemotingCommand request) throws Exception {
- final RegisterBrokerRequestHeader requestHeader = request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class);
++ final RegisterBrokerRequestHeader requestHeader = (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class);
+ final BrokerConfigLite liteConfig = new BrokerConfigLite(requestHeader.getClusterName(),
+ requestHeader.getBrokerName(),
+ requestHeader.getBrokerAddr(),
+ requestHeader.getBrokerId());
+ if (isolatedBrokers.containsKey(liteConfig)) {
+ // return response with SYSTEM_ERROR
+ return RemotingCommand.createResponseCommand(null);
+ }
+ return namesrvController.getRemotingServer().getDefaultProcessorPair().getObject1().processRequest(ctx, request);
+ }
+
+ @Override
+ public boolean rejectRequest() {
+ return false;
+ }
+ }, null);
+
+ namesrvControllers.add(namesrvController);
+ return namesrvController;
+
+ }
+
+ public static BrokerContainer createAndStartBrokerContainer(String nsAddr) {
+ BrokerContainerConfig brokerContainerConfig = new BrokerContainerConfig();
+ NettyServerConfig nettyServerConfig = new NettyServerConfig();
+ NettyClientConfig nettyClientConfig = new NettyClientConfig();
+ brokerContainerConfig.setNamesrvAddr(nsAddr);
+
+ nettyServerConfig.setListenPort(generatePort(20000, 10000));
+ BrokerContainer brokerContainer = new BrokerContainer(brokerContainerConfig, nettyServerConfig, nettyClientConfig);
+ try {
+ Assert.assertTrue(brokerContainer.initialize());
+ LOG.info("Broker container Start, listen on {}.", nettyServerConfig.getListenPort());
+ brokerContainer.start();
+ } catch (Exception e) {
+ LOG.info("Broker container start failed", e);
+ e.printStackTrace();
+ System.exit(1);
+ }
+ brokerContainerList.add(brokerContainer);
+ return brokerContainer;
+ }
+
+ private static int generatePort(int base, int range) {
+ int result = base + RANDOM.nextInt(range);
+ while (PORTS_IN_USE.contains(result) || PORTS_IN_USE.contains(result - 2)) {
+ result = base + RANDOM.nextInt(range);
+ }
+ PORTS_IN_USE.add(result);
+ PORTS_IN_USE.add(result - 2);
+ return result;
+ }
+
+ public static BrokerController createAndAddMaster(BrokerContainer brokerContainer,
+ BrokerGroupConfig brokerGroupConfig, int brokerIndex) throws Exception {
+ BrokerConfig brokerConfig = new BrokerConfig();
+ MessageStoreConfig storeConfig = new MessageStoreConfig();
+ brokerConfig.setBrokerName(BROKER_NAME_PREFIX + brokerIndex);
+ brokerConfig.setBrokerIP1("127.0.0.1");
+ brokerConfig.setBrokerIP2("127.0.0.1");
+ brokerConfig.setBrokerId(0);
+ brokerConfig.setEnablePropertyFilter(true);
+ brokerConfig.setEnableSlaveActingMaster(brokerGroupConfig.enableSlaveActingMaster);
+ brokerConfig.setEnableRemoteEscape(brokerGroupConfig.enableRemoteEscape);
+ brokerConfig.setSlaveReadEnable(brokerGroupConfig.slaveReadEnable);
+ brokerConfig.setLockInStrictMode(true);
+ brokerConfig.setConsumerOffsetUpdateVersionStep(10);
+ brokerConfig.setDelayOffsetUpdateVersionStep(10);
+ brokerConfig.setCompatibleWithOldNameSrv(false);
+ brokerConfig.setListenPort(generatePort(brokerContainer.getRemotingServer().localListenPort(), 10000));
+
+ String baseDir = createBaseDir(brokerConfig.getBrokerName() + "_" + brokerConfig.getBrokerId()).getAbsolutePath();
+ storeConfig.setStorePathRootDir(baseDir);
+ storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog");
+ storeConfig.setHaListenPort(generatePort(30000, 10000));
+ storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE);
+ storeConfig.setMaxIndexNum(INDEX_NUM);
+ storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);
+ storeConfig.setTotalReplicas(brokerGroupConfig.totalReplicas);
+ storeConfig.setInSyncReplicas(brokerGroupConfig.inSyncReplicas);
+ storeConfig.setMinInSyncReplicas(brokerGroupConfig.minReplicas);
+ storeConfig.setEnableAutoInSyncReplicas(brokerGroupConfig.autoReplicas);
+ storeConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
+ storeConfig.setSyncFlushTimeout(10 * 1000);
+
+ System.out.printf("start master %s with port %d-%d%n", brokerConfig.getCanonicalName(), brokerConfig.getListenPort(), storeConfig.getHaListenPort());
+ BrokerController brokerController = null;
+ try {
+ brokerController = brokerContainer.addBroker(brokerConfig, storeConfig);
+ Assert.assertNotNull(brokerController);
+ brokerController.start();
+ TMP_FILE_LIST.add(new File(brokerController.getTopicConfigManager().configFilePath()));
+ TMP_FILE_LIST.add(new File(brokerController.getSubscriptionGroupManager().configFilePath()));
+ LOG.info("Broker Start name:{} addr:{}", brokerConfig.getBrokerName(), brokerController.getBrokerAddr());
+ } catch (Exception e) {
+ LOG.info("Broker start failed", e);
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ return brokerController;
+ }
+
+ protected static DefaultMQProducer createProducer(String producerGroup) {
+ DefaultMQProducer producer = new DefaultMQProducer(producerGroup);
+ producer.setInstanceName(UUID.randomUUID().toString());
+ producer.setNamesrvAddr(nsAddr);
+ return producer;
+ }
+
+ protected static TransactionMQProducer createTransactionProducer(String producerGroup,
+ TransactionCheckListener transactionCheckListener) {
+ TransactionMQProducer producer = new TransactionMQProducer(producerGroup);
+ producer.setInstanceName(UUID.randomUUID().toString());
+ producer.setNamesrvAddr(nsAddr);
+ producer.setTransactionCheckListener(transactionCheckListener);
+ return producer;
+ }
+
+ protected static DefaultMQPullConsumer createPullConsumer(String consumerGroup) {
+ DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);
+ consumer.setInstanceName(UUID.randomUUID().toString());
+ consumer.setNamesrvAddr(nsAddr);
+ return consumer;
+ }
+
+ protected static DefaultMQPushConsumer createPushConsumer(String consumerGroup) {
+ DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup);
+ consumer.setInstanceName(UUID.randomUUID().toString());
+ consumer.setNamesrvAddr(nsAddr);
+ return consumer;
+ }
+
+ protected static void createAndAddSlave(int slaveBrokerId, BrokerContainer brokerContainer,
+ BrokerController master) {
+ BrokerConfig slaveBrokerConfig = new BrokerConfig();
+ slaveBrokerConfig.setBrokerName(master.getBrokerConfig().getBrokerName());
+ slaveBrokerConfig.setBrokerId(slaveBrokerId);
+ slaveBrokerConfig.setBrokerClusterName(master.getBrokerConfig().getBrokerClusterName());
+ slaveBrokerConfig.setCompatibleWithOldNameSrv(false);
+ slaveBrokerConfig.setBrokerIP1("127.0.0.1");
+ slaveBrokerConfig.setBrokerIP2("127.0.0.1");
+ slaveBrokerConfig.setEnablePropertyFilter(true);
+ slaveBrokerConfig.setSlaveReadEnable(true);
+ slaveBrokerConfig.setEnableSlaveActingMaster(true);
+ slaveBrokerConfig.setEnableRemoteEscape(true);
+ slaveBrokerConfig.setLockInStrictMode(true);
+ slaveBrokerConfig.setListenPort(generatePort(brokerContainer.getRemotingServer().localListenPort(), 10000));
+ slaveBrokerConfig.setConsumerOffsetUpdateVersionStep(10);
+ slaveBrokerConfig.setDelayOffsetUpdateVersionStep(10);
+
+ MessageStoreConfig storeConfig = slaveStoreConfigCache.get(slaveBrokerConfig);
+
+ if (storeConfig == null) {
+ storeConfig = new MessageStoreConfig();
+ String baseDir = createBaseDir(slaveBrokerConfig.getBrokerName() + "_" + slaveBrokerConfig.getBrokerId()).getAbsolutePath();
+ storeConfig.setStorePathRootDir(baseDir);
+ storeConfig.setStorePathCommitLog(baseDir + File.separator + "commitlog");
+ storeConfig.setHaListenPort(generatePort(master.getMessageStoreConfig().getHaListenPort(), 10000));
+ storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE);
+ storeConfig.setMaxIndexNum(INDEX_NUM);
+ storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);
+ storeConfig.setTotalReplicas(master.getMessageStoreConfig().getTotalReplicas());
+ storeConfig.setInSyncReplicas(master.getMessageStoreConfig().getInSyncReplicas());
+ storeConfig.setMinInSyncReplicas(master.getMessageStoreConfig().getMinInSyncReplicas());
+ storeConfig.setBrokerRole(BrokerRole.SLAVE);
+ slaveStoreConfigCache.put(slaveBrokerConfig, storeConfig);
+ }
+
+ System.out.printf("start slave %s with port %d-%d%n", slaveBrokerConfig.getCanonicalName(), slaveBrokerConfig.getListenPort(), storeConfig.getHaListenPort());
+
+ try {
+ BrokerController brokerController = brokerContainer.addBroker(slaveBrokerConfig, storeConfig);
+ Assert.assertNotNull(brokerContainer);
+ brokerController.start();
+ TMP_FILE_LIST.add(new File(brokerController.getTopicConfigManager().configFilePath()));
+ TMP_FILE_LIST.add(new File(brokerController.getSubscriptionGroupManager().configFilePath()));
+ LOG.info("Add slave name:{} addr:{}", slaveBrokerConfig.getBrokerName(), brokerController.getBrokerAddr());
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException("Couldn't add slave broker", e);
+ }
+ }
+
+ protected static void removeSlaveBroker(int slaveBrokerId, BrokerContainer brokerContainer,
+ BrokerController master) throws Exception {
+ BrokerIdentity brokerIdentity = new BrokerIdentity(master.getBrokerConfig().getBrokerClusterName(),
+ master.getBrokerConfig().getBrokerName(), slaveBrokerId);
+
+ brokerContainer.removeBroker(brokerIdentity);
+ }
+
+ protected static void awaitUntilSlaveOK() {
+ await().atMost(100, TimeUnit.SECONDS)
+ .until(() -> {
+ boolean isOk = master1With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2
+ && master1With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;
+ for (HAConnection haConnection : master1With3Replicas.getMessageStore().getHaService().getConnectionList()) {
+ isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);
+ }
+ return isOk;
+ });
+
+ await().atMost(100, TimeUnit.SECONDS)
+ .until(() -> {
+ boolean isOk = master2With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2
+ && master2With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;
+ for (HAConnection haConnection : master2With3Replicas.getMessageStore().getHaService().getConnectionList()) {
+ isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);
+ }
+ return isOk;
+ });
+
+ await().atMost(100, TimeUnit.SECONDS)
+ .until(() -> {
+ boolean isOk = master3With3Replicas.getMessageStore().getHaService().getConnectionCount().get() == 2
+ && master3With3Replicas.getMessageStore().getAliveReplicaNumInGroup() == 3;
+ for (HAConnection haConnection : master3With3Replicas.getMessageStore().getHaService().getConnectionList()) {
+ isOk &= haConnection.getCurrentState().equals(HAConnectionState.TRANSFER);
+ }
+ return isOk;
+ });
+
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ protected static void isolateBroker(BrokerController brokerController) {
+ final BrokerConfig config = brokerController.getBrokerConfig();
+
+ BrokerConfigLite liteConfig = new BrokerConfigLite(config.getBrokerClusterName(),
+ config.getBrokerName(),
+ brokerController.getBrokerAddr(),
+ config.getBrokerId());
+
+ // Reject register requests from the specific broker
+ isolatedBrokers.putIfAbsent(liteConfig, brokerController);
+
+ // UnRegister the specific broker immediately
+ namesrvController.getRouteInfoManager().unregisterBroker(liteConfig.getClusterName(),
+ liteConfig.getBrokerAddr(),
+ liteConfig.getBrokerName(),
+ liteConfig.getBrokerId());
+ }
+
+ protected static void cancelIsolatedBroker(BrokerController brokerController) {
+ final BrokerConfig config = brokerController.getBrokerConfig();
+
+ BrokerConfigLite liteConfig = new BrokerConfigLite(config.getBrokerClusterName(),
+ config.getBrokerName(),
+ brokerController.getBrokerAddr(),
+ config.getBrokerId());
+
+ isolatedBrokers.remove(liteConfig);
+ brokerController.registerBrokerAll(true, false, true);
+
+ await().atMost(Duration.ofMinutes(1)).until(() -> namesrvController.getRouteInfoManager()
+ .getBrokerMemberGroup(liteConfig.getClusterName(), liteConfig.brokerName).getBrokerAddrs()
+ .containsKey(liteConfig.getBrokerId()));
+ }
+
+ protected static InnerSalveBrokerController getSlaveFromContainerByName(BrokerContainer brokerContainer,
+ String brokerName) {
+ InnerSalveBrokerController targetSlave = null;
+ for (InnerSalveBrokerController slave : brokerContainer.getSlaveBrokers()) {
+ if (slave.getBrokerConfig().getBrokerName().equals(brokerName)) {
+ targetSlave = slave;
+ }
+ }
+
+ return targetSlave;
+ }
+
+ protected static void changeCompatibleMode(boolean compatibleMode) {
+ brokerContainer1.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));
+ brokerContainer2.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));
+ brokerContainer3.getBrokerControllers().forEach(brokerController -> brokerController.getBrokerConfig().setCompatibleWithOldNameSrv(compatibleMode));
+ }
+
+ protected static Set<MessageQueue> filterMessageQueue(Set<MessageQueue> mqSet, String topic) {
+ Set<MessageQueue> targetMqSet = new HashSet<>();
+ if (topic != null) {
+ for (MessageQueue mq : mqSet) {
+ if (mq.getTopic().equals(topic)) {
+ targetMqSet.add(mq);
+ }
+ }
+ }
+
+ return targetMqSet;
+ }
+
+ public static class BrokerGroupConfig {
+ int totalReplicas = 3;
+ int minReplicas = 1;
+ int inSyncReplicas = 2;
+ boolean autoReplicas = true;
+ boolean enableSlaveActingMaster = true;
+ boolean enableRemoteEscape = true;
+ boolean slaveReadEnable = true;
+
+ public BrokerGroupConfig() {
+ }
+
+ public BrokerGroupConfig(final int totalReplicas, final int minReplicas, final int inSyncReplicas,
+ final boolean autoReplicas, boolean enableSlaveActingMaster, boolean slaveReadEnable) {
+ this.totalReplicas = totalReplicas;
+ this.minReplicas = minReplicas;
+ this.inSyncReplicas = inSyncReplicas;
+ this.autoReplicas = autoReplicas;
+ this.enableSlaveActingMaster = enableSlaveActingMaster;
+ this.slaveReadEnable = slaveReadEnable;
+ }
+ }
+
+ static class BrokerConfigLite {
+ private String clusterName;
+ private String brokerName;
+ private String brokerAddr;
+ private long brokerId;
+
+ public BrokerConfigLite(final String clusterName, final String brokerName, final String brokerAddr,
+ final long brokerId) {
+ this.clusterName = clusterName;
+ this.brokerName = brokerName;
+ this.brokerAddr = brokerAddr;
+ this.brokerId = brokerId;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public String getBrokerName() {
+ return brokerName;
+ }
+
+ public String getBrokerAddr() {
+ return brokerAddr;
+ }
+
+ public long getBrokerId() {
+ return brokerId;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ final BrokerConfigLite lite = (BrokerConfigLite) o;
+
+ return new EqualsBuilder()
+ .append(clusterName, lite.clusterName)
+ .append(brokerName, lite.brokerName)
+ .append(brokerAddr, lite.brokerAddr)
+ .append(brokerId, lite.brokerId)
+ .isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 37)
+ .append(clusterName)
+ .append(brokerName)
+ .append(brokerAddr)
+ .append(brokerId)
+ .toHashCode();
+ }
+ }
+ }
diff --cc test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
index 1550cf949,1550cf949..ce2829d28
--- a/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java
@@@ -19,7 -19,7 +19,6 @@@ package org.apache.rocketmq.test.delay
import java.util.List;
import org.apache.log4j.Logger;
--import org.apache.rocketmq.test.client.consumer.balance.NormalMsgStaticBalanceIT;
import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
import org.apache.rocketmq.test.factory.MQMessageFactory;
diff --cc test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java
index 000000000,41c9c7e4f..d2f7e81ba
mode 000000,100644..100644
--- a/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java
@@@ -1,0 -1,522 +1,521 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.rocketmq.test.statictopic;
+
+ import com.google.common.collect.ImmutableList;
+ import com.google.common.collect.ImmutableSet;
+ import org.apache.log4j.Logger;
+ import org.apache.rocketmq.broker.BrokerController;
+ import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+ import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+ import org.apache.rocketmq.client.producer.DefaultMQProducer;
+ import org.apache.rocketmq.common.MixAll;
+ import org.apache.rocketmq.common.admin.ConsumeStats;
+ import org.apache.rocketmq.common.admin.OffsetWrapper;
+ import org.apache.rocketmq.common.admin.TopicStatsTable;
+ import org.apache.rocketmq.common.message.MessageExt;
+ import org.apache.rocketmq.common.message.MessageQueue;
+ import org.apache.rocketmq.common.rpc.ClientMetadata;
+ import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
+ import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingOne;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
+ import org.apache.rocketmq.test.base.BaseConf;
+ import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
+ import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
+ import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
+ import org.apache.rocketmq.test.util.MQAdminTestUtils;
+ import org.apache.rocketmq.test.util.MQRandomUtils;
+ import org.apache.rocketmq.test.util.TestUtils;
+ import org.apache.rocketmq.test.util.VerifyUtils;
+ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+ import org.apache.rocketmq.tools.admin.MQAdminUtils;
+ import org.junit.After;
+ import org.junit.Assert;
+ import org.junit.Before;
+ import org.junit.FixMethodOrder;
-import org.junit.Ignore;
+ import org.junit.Test;
+
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.Comparator;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+
+ import static com.google.common.truth.Truth.assertThat;
+ import static org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils.getMappingDetailFromConfig;
+
+ @FixMethodOrder
+ public class StaticTopicIT extends BaseConf {
+
+ private static Logger logger = Logger.getLogger(StaticTopicIT.class);
+ private DefaultMQAdminExt defaultMQAdminExt;
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty("rocketmq.client.rebalance.waitInterval", "500");
+ defaultMQAdminExt = getAdmin(nsAddr);
+ waitBrokerRegistered(nsAddr, clusterName, brokerNum);
+ defaultMQAdminExt.start();
+ }
+
+
+ @Test
+ public void testCommandsWithCluster() throws Exception {
+ //This case is used to mock the env to test the command manually
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener());
+ int queueNum = 10;
+ int msgEachQueue = 100;
+
+ {
+ MQAdminTestUtils.createStaticTopicWithCommand(topic, queueNum, null, clusterName, nsAddr);
+ sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 0);
+ //consume and check
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+ {
+ MQAdminTestUtils.remappingStaticTopicWithCommand(topic, null, clusterName, nsAddr);
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);
+ sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, msgEachQueue);
+ }
+ }
+
+ @Test
+ public void testCommandsWithBrokers() throws Exception {
+ //This case is used to mock the env to test the command manually
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener());
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ {
+ Set<String> brokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopicWithCommand(topic, queueNum, brokers, null, nsAddr);
+ sendMessagesAndCheck(producer, brokers, topic, queueNum, msgEachQueue, 0);
+ //consume and check
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+ {
+ Set<String> brokers = ImmutableSet.of(broker2Name);
+ MQAdminTestUtils.remappingStaticTopicWithCommand(topic, brokers, null, nsAddr);
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);
+ sendMessagesAndCheck(producer, brokers, topic, queueNum, msgEachQueue, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 2);
+ }
+ }
+
+ @Test
+ public void testNoTargetBrokers() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ int queueNum = 10;
+ {
+ Set<String> targetBrokers = new HashSet<>();
+ targetBrokers.add(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size());
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ TopicConfigAndQueueMapping configMapping = remoteBrokerConfigMap.get(broker2Name);
+ Assert.assertEquals(0, configMapping.getWriteQueueNums());
+ Assert.assertEquals(0, configMapping.getReadQueueNums());
+ Assert.assertEquals(0, configMapping.getMappingDetail().getHostedQueues().size());
+ }
+
+ {
+ Set<String> targetBrokers = new HashSet<>();
+ targetBrokers.add(broker2Name);
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size());
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ }
+
+ }
+
+ private void sendMessagesAndCheck(RMQNormalProducer producer, Set<String> targetBrokers, String topic, int queueNum, int msgEachQueue, long baseOffset) throws Exception {
+ ClientMetadata clientMetadata = MQAdminUtils.getBrokerAndTopicMetadata(topic, defaultMQAdminExt);
+ List<MessageQueue> messageQueueList = producer.getMessageQueue();
+ Assert.assertEquals(queueNum, messageQueueList.size());
+ for (int i = 0; i < queueNum; i++) {
+ MessageQueue messageQueue = messageQueueList.get(i);
+ Assert.assertEquals(topic, messageQueue.getTopic());
+ Assert.assertEquals(TopicQueueMappingUtils.getMockBrokerName(MixAll.METADATA_SCOPE_GLOBAL), messageQueue.getBrokerName());
+ Assert.assertEquals(i, messageQueue.getQueueId());
+ String destBrokerName = clientMetadata.getBrokerNameFromMessageQueue(messageQueue);
+ Assert.assertTrue(targetBrokers.contains(destBrokerName));
+ }
+ for(MessageQueue messageQueue: messageQueueList) {
+ producer.send(msgEachQueue, messageQueue);
+ }
+ Assert.assertEquals(0, producer.getSendErrorMsg().size());
+ //leave the time to build the cq
+ Assert.assertTrue(awaitDispatchMs(500));
+ for(MessageQueue messageQueue: messageQueueList) {
+ Assert.assertEquals(0, defaultMQAdminExt.minOffset(messageQueue));
+ Assert.assertEquals(msgEachQueue + baseOffset, defaultMQAdminExt.maxOffset(messageQueue));
+ }
+ TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic);
+ for(MessageQueue messageQueue: messageQueueList) {
+ Assert.assertEquals(0, topicStatsTable.getOffsetTable().get(messageQueue).getMinOffset());
+ Assert.assertEquals(msgEachQueue + baseOffset, topicStatsTable.getOffsetTable().get(messageQueue).getMaxOffset());
+ }
+ }
+
+ private Map<Integer, List<MessageExt>> computeMessageByQueue(Collection<Object> msgs) {
+ Map<Integer, List<MessageExt>> messagesByQueue = new HashMap<>();
+ for (Object object : msgs) {
+ MessageExt messageExt = (MessageExt) object;
+ if (!messagesByQueue.containsKey(messageExt.getQueueId())) {
+ messagesByQueue.put(messageExt.getQueueId(), new ArrayList<>());
+ }
+ messagesByQueue.get(messageExt.getQueueId()).add(messageExt);
+ }
+ for (List<MessageExt> msgEachQueue: messagesByQueue.values()) {
+ Collections.sort(msgEachQueue, new Comparator<MessageExt>() {
+ @Override
+ public int compare(MessageExt o1, MessageExt o2) {
+ return (int) (o1.getQueueOffset() - o2.getQueueOffset());
+ }
+ });
+ }
+ return messagesByQueue;
+ }
+
+ private void consumeMessagesAndCheck(RMQNormalProducer producer, RMQNormalConsumer consumer, String topic, int queueNum, int msgEachQueue, int startGen, int genNum) {
+ consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), 30000);
- /*System.out.println("produce:" + producer.getAllMsgBody().size());
- System.out.println("consume:" + consumer.getListener().getAllMsgBody().size());*/
++// System.out.println("produce:" + producer.getAllMsgBody().size());
++// System.out.println("consume:" + consumer.getListener().getAllMsgBody().size());
+
+ Assert.assertEquals(producer.getAllMsgBody().size(), consumer.getListener().getAllMsgBody().size());
+ assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(),
+ consumer.getListener().getAllMsgBody()))
+ .containsExactlyElementsIn(producer.getAllMsgBody());
+ Map<Integer, List<MessageExt>> messagesByQueue = computeMessageByQueue(consumer.getListener().getAllOriginMsg());
+ Assert.assertEquals(queueNum, messagesByQueue.size());
+ for (int i = 0; i < queueNum; i++) {
+ List<MessageExt> messageExts = messagesByQueue.get(i);
+ /*for (MessageExt messageExt:messageExts) {
+ System.out.printf("%d %d\n", messageExt.getQueueId(), messageExt.getQueueOffset());
+ }*/
+ int totalEachQueue = msgEachQueue * genNum;
+ Assert.assertEquals(totalEachQueue, messageExts.size());
+ for (int j = 0; j < totalEachQueue; j++) {
+ MessageExt messageExt = messageExts.get(j);
+ int currGen = startGen + j / msgEachQueue;
+ Assert.assertEquals(topic, messageExt.getTopic());
+ Assert.assertEquals(TopicQueueMappingUtils.getMockBrokerName(MixAll.METADATA_SCOPE_GLOBAL), messageExt.getBrokerName());
+ Assert.assertEquals(i, messageExt.getQueueId());
+ Assert.assertEquals((j % msgEachQueue) + currGen * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, messageExt.getQueueOffset());
+ }
+ }
+ }
+
+
+ @Test
+ public void testCreateProduceConsumeStaticTopic() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener());
+
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ //create static topic
+ Map<String, TopicConfigAndQueueMapping> localBrokerConfigMap = MQAdminTestUtils.createStaticTopic(topic, queueNum, getBrokers(), defaultMQAdminExt);
+ //check the static topic config
+ {
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size());
+ for (Map.Entry<String, TopicConfigAndQueueMapping> entry: remoteBrokerConfigMap.entrySet()) {
+ String broker = entry.getKey();
+ TopicConfigAndQueueMapping configMapping = entry.getValue();
+ TopicConfigAndQueueMapping localConfigMapping = localBrokerConfigMap.get(broker);
+ Assert.assertNotNull(localConfigMapping);
+ Assert.assertEquals(configMapping, localConfigMapping);
+ }
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ }
+ //send and check
+ sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 0);
+ //consume and check
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+
+
+ @Test
+ public void testRemappingProduceConsumeStaticTopic() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener());
+
+ int queueNum = 1;
+ int msgEachQueue = 100;
+ //create send consume
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+ //remapping the static topic
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker2Name);
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ for (TopicQueueMappingOne mappingOne: globalIdMap.values()) {
+ Assert.assertEquals(broker2Name, mappingOne.getBname());
+ Assert.assertEquals(TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, mappingOne.getItems().get(mappingOne.getItems().size() - 1).getLogicOffset());
+ }
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), consumer.getConsumer(), defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 2);
+ }
+ }
+
+
+ public boolean awaitRefreshStaticTopicMetadata(long timeMs, String topic, DefaultMQProducer producer, DefaultMQPushConsumer consumer, DefaultMQAdminExt adminExt) throws Exception {
+ long start = System.currentTimeMillis();
+ MQClientInstance currentInstance = null;
+ while (System.currentTimeMillis() - start <= timeMs) {
+ boolean allOk = true;
+ if (producer != null) {
+ currentInstance = producer.getDefaultMQProducerImpl().getmQClientFactory();
+ currentInstance.updateTopicRouteInfoFromNameServer(topic);
+ if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {
+ allOk = false;
+ }
+ }
+ if (consumer != null) {
+ currentInstance = consumer.getDefaultMQPushConsumerImpl().getmQClientFactory();
+ currentInstance.updateTopicRouteInfoFromNameServer(topic);
+ if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {
+ allOk = false;
+ }
+ }
+ if (adminExt != null) {
+ currentInstance = adminExt.getDefaultMQAdminExtImpl().getMqClientInstance();
+ currentInstance.updateTopicRouteInfoFromNameServer(topic);
+ if (!MQAdminTestUtils.checkStaticTopic(topic, adminExt, currentInstance)) {
+ allOk = false;
+ }
+ }
+ if (allOk) {
+ return true;
+ }
+ Thread.sleep(100);
+ }
+ return false;
+ }
+
+
+ @Test
+ public void testDoubleReadCheckConsumerOffset() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ String group = initConsumerGroup();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ RMQNormalConsumer consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener());
+ long start = System.currentTimeMillis();
+
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ //create static topic
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1);
+ }
+ producer.shutdown();
+ consumer.shutdown();
+ //use a new producer
+ producer = getProducer(nsAddr, topic);
+
+ ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(group);
+ List<MessageQueue> messageQueues = producer.getMessageQueue();
+ for (MessageQueue queue: messageQueues) {
+ OffsetWrapper wrapper = consumeStats.getOffsetTable().get(queue);
+ Assert.assertNotNull(wrapper);
+ Assert.assertEquals(msgEachQueue, wrapper.getBrokerOffset());
+ Assert.assertEquals(msgEachQueue, wrapper.getConsumerOffset());
+ Assert.assertTrue(wrapper.getLastTimestamp() > start);
+ }
+
+ List<String> brokers = ImmutableList.of(broker2Name, broker3Name, broker1Name);
+ for (int i = 0; i < brokers.size(); i++) {
+ Set<String> targetBrokers = ImmutableSet.of(brokers.get(i));
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ //make the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, (i + 1) * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ }
+
+ TestUtils.waitForSeconds(1);
+ consumeStats = defaultMQAdminExt.examineConsumeStats(group);
+
+ messageQueues = producer.getMessageQueue();
+ for (MessageQueue queue: messageQueues) {
+ OffsetWrapper wrapper = consumeStats.getOffsetTable().get(queue);
+ Assert.assertNotNull(wrapper);
+ Assert.assertEquals(msgEachQueue + brokers.size() * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, wrapper.getBrokerOffset());
+ Assert.assertEquals(msgEachQueue, wrapper.getConsumerOffset());
+ Assert.assertTrue(wrapper.getLastTimestamp() > start);
+ }
+ consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener());
+ consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 1, brokers.size());
+ }
+
+
+ @Test
+ public void testRemappingAndClear() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ //create to broker1Name
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ //leave the time to refresh the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ }
+
+ //remapping to broker2Name
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker2Name);
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ //leave the time to refresh the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 1 * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ }
+
+ //remapping to broker3Name
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker3Name);
+ MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt);
+ //leave the time to refresh the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 2 * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE);
+ }
+
+ // 1 -> 2 -> 3, currently 1 should not has any mappings
+
+ {
+ for (int i = 0; i < 10; i++) {
+ for (BrokerController brokerController: brokerControllerList) {
+ brokerController.getTopicQueueMappingCleanService().wakeup();
+ }
+ Thread.sleep(100);
+ }
+ Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, brokerConfigMap.size());
+ TopicConfigAndQueueMapping config1 = brokerConfigMap.get(broker1Name);
+ TopicConfigAndQueueMapping config2 = brokerConfigMap.get(broker2Name);
+ TopicConfigAndQueueMapping config3 = brokerConfigMap.get(broker3Name);
+ Assert.assertEquals(0, config1.getMappingDetail().getHostedQueues().size());
+ Assert.assertEquals(queueNum, config2.getMappingDetail().getHostedQueues().size());
+
+ Assert.assertEquals(queueNum, config3.getMappingDetail().getHostedQueues().size());
+
+ }
+ {
+ Set<String> topics = new HashSet<>(brokerController1.getTopicConfigManager().getTopicConfigTable().keySet());
+ topics.remove(topic);
+ brokerController1.getMessageStore().cleanUnusedTopic(topics);
+ brokerController2.getMessageStore().cleanUnusedTopic(topics);
+ for (int i = 0; i < 10; i++) {
+ for (BrokerController brokerController: brokerControllerList) {
+ brokerController.getTopicQueueMappingCleanService().wakeup();
+ }
+ Thread.sleep(100);
+ }
+
+ Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ Assert.assertEquals(brokerNum, brokerConfigMap.size());
+ TopicConfigAndQueueMapping config1 = brokerConfigMap.get(broker1Name);
+ TopicConfigAndQueueMapping config2 = brokerConfigMap.get(broker2Name);
+ TopicConfigAndQueueMapping config3 = brokerConfigMap.get(broker3Name);
+ Assert.assertEquals(0, config1.getMappingDetail().getHostedQueues().size());
+ Assert.assertEquals(queueNum, config2.getMappingDetail().getHostedQueues().size());
+ Assert.assertEquals(queueNum, config3.getMappingDetail().getHostedQueues().size());
+ //The first leader will clear it
+ for (List<LogicQueueMappingItem> items : config1.getMappingDetail().getHostedQueues().values()) {
+ Assert.assertEquals(3, items.size());
+ }
+ //The second leader do nothing
+ for (List<LogicQueueMappingItem> items : config3.getMappingDetail().getHostedQueues().values()) {
+ Assert.assertEquals(1, items.size());
+ }
+ }
+ }
+
+
+ @Test
+ public void testRemappingWithNegativeLogicOffset() throws Exception {
+ String topic = "static" + MQRandomUtils.getRandomTopic();
+ RMQNormalProducer producer = getProducer(nsAddr, topic);
+ int queueNum = 10;
+ int msgEachQueue = 100;
+ //create and send
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker1Name);
+ MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt);
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ }
+
+ //remapping the static topic with -1 logic offset
+ {
+ Set<String> targetBrokers = ImmutableSet.of(broker2Name);
+ MQAdminTestUtils.remappingStaticTopicWithNegativeLogicOffset(topic, targetBrokers, defaultMQAdminExt);
+ Map<String, TopicConfigAndQueueMapping> remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
+ TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap);
+ Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true);
+ Assert.assertEquals(queueNum, globalIdMap.size());
+ for (TopicQueueMappingOne mappingOne: globalIdMap.values()) {
+ Assert.assertEquals(broker2Name, mappingOne.getBname());
+ Assert.assertEquals(-1, mappingOne.getItems().get(mappingOne.getItems().size() - 1).getLogicOffset());
+ }
+ //leave the time to refresh the metadata
+ awaitRefreshStaticTopicMetadata(3000, topic, producer.getProducer(), null, defaultMQAdminExt);
+ //here the gen should be 0
+ sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0);
+ }
+ }
+
+
+ @After
+ public void tearDown() {
+ System.setProperty("rocketmq.client.rebalance.waitInterval", "20000");
+ super.shutdown();
+ }
+
+ }
diff --cc tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
index 2206bc175,b8bb39979..311beb7f8
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
@@@ -41,9 -37,9 +37,10 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.body.ConsumerConnection;
import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.GroupList;
+ import org.apache.rocketmq.common.protocol.body.HARuntimeInfo;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
@@@ -291,11 -318,11 +324,16 @@@ public class DefaultMQAdminExt extends
return defaultMQAdminExtImpl.examineProducerConnectionInfo(producerGroup, topic);
}
+ @Override
+ public ProducerTableInfo getAllProducerInfo(final String brokerAddr) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
+ return defaultMQAdminExtImpl.getAllProducerInfo(brokerAddr);
+ }
+
+ @Override public void deleteTopicInNameServer(Set<String> addrs, String clusterName,
+ String topic) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+ defaultMQAdminExtImpl.deleteTopicInNameServer(addrs, clusterName, topic);
+ }
+
@Override
public List<String> getNameServerAddressList() {
return this.defaultMQAdminExtImpl.getNameServerAddressList();
diff --cc tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
index 9e99925a8,7b78ead3a..79f369b1e
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
@@@ -78,7 -88,11 +90,10 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+ import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+ import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+ import org.apache.rocketmq.common.subscription.GroupForbidden;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
-import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.common.RemotingUtil;
@@@ -213,16 -259,10 +260,15 @@@ public class DefaultMQAdminExtImpl impl
@Override public void updateGlobalWhiteAddrConfig(String addr,
String globalWhiteAddrs) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
- this.mqClientInstance.getMQClientAPIImpl().updateGlobalWhiteAddrsConfig(addr, globalWhiteAddrs, timeoutMillis);
+ this.mqClientInstance.getMQClientAPIImpl().updateGlobalWhiteAddrsConfig(addr, globalWhiteAddrs, null, timeoutMillis);
+ }
+
+ @Override public void updateGlobalWhiteAddrConfig(String addr,
+ String globalWhiteAddrs, String aclFileFullPath) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+ this.mqClientInstance.getMQClientAPIImpl().updateGlobalWhiteAddrsConfig(addr, globalWhiteAddrs, aclFileFullPath, timeoutMillis);
}
- @Override
- public ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
+ @Override public ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
String addr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
return this.mqClientInstance.getMQClientAPIImpl().getBrokerClusterAclInfo(addr, timeoutMillis);
}
@@@ -754,45 -1085,8 +1097,44 @@@
return result;
}
+ @Override
+ public boolean deleteExpiredCommitLog(String cluster) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException {
+ boolean result = false;
+ try {
+ ClusterInfo clusterInfo = examineBrokerClusterInfo();
+ if (null == cluster || "".equals(cluster)) {
+ for (String targetCluster : clusterInfo.retrieveAllClusterNames()) {
+ result = deleteExpiredCommitLogByCluster(clusterInfo, targetCluster);
+ }
+ } else {
+ result = deleteExpiredCommitLogByCluster(clusterInfo, cluster);
+ }
+ } catch (MQBrokerException e) {
+ log.error("deleteExpiredCommitLog error.", e);
+ }
+
+ return result;
+ }
+
+ public boolean deleteExpiredCommitLogByCluster(ClusterInfo clusterInfo, String cluster) throws RemotingConnectException,
+ RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException {
+ boolean result = false;
+ String[] addrs = clusterInfo.retrieveAllAddrByCluster(cluster);
+ for (String addr : addrs) {
+ result = deleteExpiredCommitLogByAddr(addr);
+ }
+ return result;
+ }
+
+ @Override
+ public boolean deleteExpiredCommitLogByAddr(String addr) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException {
+ boolean result = mqClientInstance.getMQClientAPIImpl().deleteExpiredCommitLog(addr, timeoutMillis);
+ log.warn("Delete expired CommitLog on target " + addr + " broker " + result);
+ return result;
+ }
+
- @Override
- public boolean cleanUnusedTopic(String cluster) throws RemotingConnectException, RemotingSendRequestException,
- RemotingTimeoutException, MQClientException, InterruptedException {
+ @Override public boolean cleanUnusedTopic(
+ String cluster) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException {
boolean result = false;
try {
ClusterInfo clusterInfo = examineBrokerClusterInfo();
diff --cc tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
index 58e3fb713,5ed147c13..59ac6c5cb
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
@@@ -40,9 -36,9 +36,10 @@@ import org.apache.rocketmq.common.proto
import org.apache.rocketmq.common.protocol.body.ConsumerConnection;
import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.GroupList;
+ import org.apache.rocketmq.common.protocol.body.HARuntimeInfo;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
+import org.apache.rocketmq.common.protocol.body.ProducerTableInfo;
import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
@@@ -78,13 -92,12 +93,15 @@@ public interface MQAdminExt extends MQA
void deletePlainAccessConfig(final String addr, final String accessKey) throws RemotingException, MQBrokerException,
InterruptedException, MQClientException;
- void updateGlobalWhiteAddrConfig(final String addr, final String globalWhiteAddrs)throws RemotingException, MQBrokerException,
+ void updateGlobalWhiteAddrConfig(final String addr,
+ final String globalWhiteAddrs) throws RemotingException, MQBrokerException,
InterruptedException, MQClientException;
+ void updateGlobalWhiteAddrConfig(final String addr, final String globalWhiteAddrs, String aclFileFullPath)throws RemotingException, MQBrokerException,
+ InterruptedException, MQClientException;
+
- ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(final String addr) throws RemotingException, MQBrokerException,
+ ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
+ final String addr) throws RemotingException, MQBrokerException,
InterruptedException, MQClientException;
AclConfig examineBrokerClusterAclConfig(final String addr) throws RemotingException, MQBrokerException,
diff --cc tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
index c69b7bf9e,bc720383b..dbf1a8eeb
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
@@@ -37,8 -37,8 +37,9 @@@ import org.apache.rocketmq.tools.comman
import org.apache.rocketmq.tools.command.broker.BrokerStatusSubCommand;
import org.apache.rocketmq.tools.command.broker.CleanExpiredCQSubCommand;
import org.apache.rocketmq.tools.command.broker.CleanUnusedTopicCommand;
+import org.apache.rocketmq.tools.command.broker.DeleteExpiredCommitLogSubCommand;
import org.apache.rocketmq.tools.command.broker.GetBrokerConfigCommand;
+ import org.apache.rocketmq.tools.command.broker.ResetMasterFlushOffsetSubCommand;
import org.apache.rocketmq.tools.command.broker.SendMsgStatusCommand;
import org.apache.rocketmq.tools.command.broker.UpdateBrokerConfigSubCommand;
import org.apache.rocketmq.tools.command.cluster.CLusterSendMsgRTCommand;
@@@ -49,11 -49,15 +50,15 @@@ import org.apache.rocketmq.tools.comman
import org.apache.rocketmq.tools.command.consumer.ConsumerStatusSubCommand;
import org.apache.rocketmq.tools.command.consumer.DeleteSubscriptionGroupCommand;
import org.apache.rocketmq.tools.command.consumer.GetConsumerConfigSubCommand;
+ import org.apache.rocketmq.tools.command.consumer.SetConsumeModeSubCommand;
import org.apache.rocketmq.tools.command.consumer.StartMonitoringSubCommand;
import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupSubCommand;
-import org.apache.rocketmq.tools.command.container.AddBrokerSubCommand;
-import org.apache.rocketmq.tools.command.container.RemoveBrokerSubCommand;
import org.apache.rocketmq.tools.command.export.ExportConfigsCommand;
import org.apache.rocketmq.tools.command.export.ExportMetadataCommand;
import org.apache.rocketmq.tools.command.export.ExportMetricsCommand;
++import org.apache.rocketmq.tools.command.container.AddBrokerSubCommand;
++import org.apache.rocketmq.tools.command.container.RemoveBrokerSubCommand;
+ import org.apache.rocketmq.tools.command.ha.HAStatusSubCommand;
import org.apache.rocketmq.tools.command.message.CheckMsgSendRTCommand;
import org.apache.rocketmq.tools.command.message.ConsumeMessageCommand;
import org.apache.rocketmq.tools.command.message.PrintMessageByQueueCommand;
diff --cc tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
index febc80e44,59df145eb..17e04e7f5
--- a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
+++ b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
@@@ -22,9 -34,10 +22,9 @@@ import org.apache.rocketmq.client.excep
import org.apache.rocketmq.client.impl.MQClientAPIImpl;
import org.apache.rocketmq.client.impl.MQClientManager;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.admin.ConsumeStats;
--import org.apache.rocketmq.common.admin.OffsetWrapper;
import org.apache.rocketmq.common.admin.TopicOffset;
import org.apache.rocketmq.common.admin.TopicStatsTable;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
@@@ -59,30 -71,18 +60,33 @@@ import org.apache.rocketmq.remoting.exc
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
import org.apache.rocketmq.tools.admin.api.MessageTrack;
+ import org.assertj.core.util.Maps;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+ import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.anyBoolean;
+ import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;