You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by di...@apache.org on 2022/04/11 02:50:45 UTC
[rocketmq-client-go] branch feature created (now 4999236)
This is an automated email from the ASF dual-hosted git repository.
dinglei pushed a change to branch feature
in repository https://gitbox.apache.org/repos/asf/rocketmq-client-go.git
at 4999236 init new empty branch feature.
This branch includes the following new commits:
new 4999236 init new empty branch feature.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[rocketmq-client-go] 01/01: init new empty branch feature.
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dinglei pushed a commit to branch feature
in repository https://gitbox.apache.org/repos/asf/rocketmq-client-go.git
commit 4999236ed73d5972fd23b1f766503aa1d9608717
Author: ShannonDing <li...@163.com>
AuthorDate: Mon Apr 11 10:50:15 2022 +0800
init new empty branch feature.
---
admin/admin.go | 210 -----
admin/option.go | 131 ---
api.go | 137 ---
benchmark/consumer.go | 277 ------
benchmark/main.go | 76 --
benchmark/message.go | 34 -
benchmark/producer.go | 290 -------
benchmark/stable.go | 280 -------
config.go | 17 -
consumer/consumer.go | 1063 -----------------------
consumer/consumer_test.go | 171 ----
consumer/interceptor.go | 115 ---
consumer/lock.go | 37 -
consumer/mock_offset_store.go | 94 ---
consumer/offset_store.go | 402 ---------
consumer/offset_store_test.go | 243 ------
consumer/option.go | 276 ------
consumer/process_queue.go | 397 ---------
consumer/pull_consumer.go | 256 ------
consumer/pull_consumer_test.go | 18 -
consumer/push_consumer.go | 1254 ----------------------------
consumer/push_consumer_test.go | 96 ---
consumer/statistics.go | 485 -----------
consumer/statistics_test.go | 214 -----
consumer/strategy.go | 247 ------
consumer/strategy_test.go | 487 -----------
docs/Introduction.md | 130 ---
docs/client-design.gliffy | 1 -
docs/feature.md | 86 --
docs/images/client-design.png | Bin 30513 -> 0 bytes
docs/zh/native-design_zh.md | 114 ---
docs/zh/rocketmq-protocol_zh.md | 117 ---
errors/errors.go | 51 --
examples/admin/topic/main.go | 64 --
examples/consumer/acl/main.go | 64 --
examples/consumer/broadcast/main.go | 57 --
examples/consumer/delay/main.go | 60 --
examples/consumer/interceptor/main.go | 83 --
examples/consumer/namespace/main.go | 65 --
examples/consumer/orderly/main.go | 60 --
examples/consumer/pull/main.go | 72 --
examples/consumer/retry/concurrent/main.go | 81 --
examples/consumer/retry/order/main.go | 76 --
examples/consumer/simple/main.go | 58 --
examples/consumer/strategy/main.go | 56 --
examples/consumer/tag/main.go | 58 --
examples/consumer/trace/main.go | 63 --
examples/producer/acl/main.go | 65 --
examples/producer/async/main.go | 65 --
examples/producer/batch/main.go | 58 --
examples/producer/delay/main.go | 55 --
examples/producer/interceptor/main.go | 74 --
examples/producer/namespace/main.go | 66 --
examples/producer/simple/main.go | 61 --
examples/producer/tag/main.go | 58 --
examples/producer/trace/main.go | 64 --
examples/producer/transaction/main.go | 106 ---
go.mod | 17 -
go.sum | 63 --
internal/callback.go | 31 -
internal/client.go | 893 --------------------
internal/constants.go | 29 -
internal/mock_client.go | 430 ----------
internal/mock_namesrv.go | 193 -----
internal/model.go | 406 ---------
internal/model_test.go | 484 -----------
internal/mq_version.go | 22 -
internal/namesrv.go | 185 ----
internal/namesrv_test.go | 149 ----
internal/perm.go | 58 --
internal/remote/codec.go | 521 ------------
internal/remote/codec_test.go | 378 ---------
internal/remote/future.go | 68 --
internal/remote/interceptor.go | 83 --
internal/remote/interceptor_test.go | 29 -
internal/remote/mock_remote_client.go | 123 ---
internal/remote/remote_client.go | 316 -------
internal/remote/remote_client_test.go | 372 ---------
internal/remote/rpchook.go | 23 -
internal/remote/tcp_conn.go | 61 --
internal/request.go | 480 -----------
internal/response.go | 49 --
internal/route.go | 668 ---------------
internal/route_test.go | 137 ---
internal/trace.go | 526 ------------
internal/trace_test.go | 125 ---
internal/transaction.go | 21 -
internal/utils/compression.go | 86 --
internal/utils/compression_test.go | 97 ---
internal/utils/errors.go | 30 -
internal/utils/files.go | 89 --
internal/utils/math.go | 32 -
internal/utils/net.go | 70 --
internal/utils/net_test.go | 24 -
internal/utils/set.go | 103 ---
internal/utils/string.go | 30 -
internal/validators.go | 43 -
primitive/auth.go | 28 -
primitive/base.go | 129 ---
primitive/base_test.go | 76 --
primitive/ctx.go | 173 ----
primitive/errors.go | 61 --
primitive/interceptor.go | 51 --
primitive/message.go | 569 -------------
primitive/message_test.go | 38 -
primitive/nsresolver.go | 223 -----
primitive/nsresolver_test.go | 136 ---
primitive/pool.go | 57 --
primitive/result.go | 128 ---
primitive/result_test.go | 74 --
primitive/trace.go | 28 -
producer/interceptor.go | 103 ---
producer/option.go | 164 ----
producer/producer.go | 602 -------------
producer/producer_test.go | 358 --------
producer/selector.go | 122 ---
producer/selector_test.go | 78 --
rlog/log.go | 180 ----
118 files changed, 20317 deletions(-)
diff --git a/admin/admin.go b/admin/admin.go
deleted file mode 100644
index 06908f4..0000000
--- a/admin/admin.go
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package admin
-
-import (
- "context"
- "fmt"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type Admin interface {
- CreateTopic(ctx context.Context, opts ...OptionCreate) error
- DeleteTopic(ctx context.Context, opts ...OptionDelete) error
- //TODO
- //TopicList(ctx context.Context, mq *primitive.MessageQueue) (*remote.RemotingCommand, error)
- //GetBrokerClusterInfo(ctx context.Context) (*remote.RemotingCommand, error)
- Close() error
-}
-
-// TODO: move outdated context to ctx
-type adminOptions struct {
- internal.ClientOptions
-}
-
-type AdminOption func(options *adminOptions)
-
-func defaultAdminOptions() *adminOptions {
- opts := &adminOptions{
- ClientOptions: internal.DefaultClientOptions(),
- }
- opts.GroupName = "TOOLS_ADMIN"
- opts.InstanceName = time.Now().String()
- return opts
-}
-
-// WithResolver nameserver resolver to fetch nameserver addr
-func WithResolver(resolver primitive.NsResolver) AdminOption {
- return func(options *adminOptions) {
- options.Resolver = resolver
- }
-}
-
-type admin struct {
- cli internal.RMQClient
-
- opts *adminOptions
-
- closeOnce sync.Once
-}
-
-// NewAdmin initialize admin
-func NewAdmin(opts ...AdminOption) (Admin, error) {
- defaultOpts := defaultAdminOptions()
- for _, opt := range opts {
- opt(defaultOpts)
- }
- namesrv, err := internal.NewNamesrv(defaultOpts.Resolver)
- defaultOpts.Namesrv = namesrv
- if err != nil {
- return nil, err
- }
-
- cli := internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil)
- if cli == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = cli.GetNameSrv()
- //log.Printf("Client: %#v", namesrv.srvs)
- return &admin{
- cli: cli,
- opts: defaultOpts,
- }, nil
-}
-
-// CreateTopic create topic.
-// TODO: another implementation like sarama, without brokerAddr as input
-func (a *admin) CreateTopic(ctx context.Context, opts ...OptionCreate) error {
- cfg := defaultTopicConfigCreate()
- for _, apply := range opts {
- apply(&cfg)
- }
-
- request := &internal.CreateTopicRequestHeader{
- Topic: cfg.Topic,
- DefaultTopic: cfg.DefaultTopic,
- ReadQueueNums: cfg.ReadQueueNums,
- WriteQueueNums: cfg.WriteQueueNums,
- Perm: cfg.Perm,
- TopicFilterType: cfg.TopicFilterType,
- TopicSysFlag: cfg.TopicSysFlag,
- Order: cfg.Order,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqCreateTopic, request, nil)
- _, err := a.cli.InvokeSync(ctx, cfg.BrokerAddr, cmd, 5*time.Second)
- if err != nil {
- rlog.Error("create topic error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- } else {
- rlog.Info("create topic success", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- })
- }
- return err
-}
-
-// DeleteTopicInBroker delete topic in broker.
-func (a *admin) deleteTopicInBroker(ctx context.Context, topic string, brokerAddr string) (*remote.RemotingCommand, error) {
- request := &internal.DeleteTopicRequestHeader{
- Topic: topic,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqDeleteTopicInBroker, request, nil)
- return a.cli.InvokeSync(ctx, brokerAddr, cmd, 5*time.Second)
-}
-
-// DeleteTopicInNameServer delete topic in nameserver.
-func (a *admin) deleteTopicInNameServer(ctx context.Context, topic string, nameSrvAddr string) (*remote.RemotingCommand, error) {
- request := &internal.DeleteTopicRequestHeader{
- Topic: topic,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqDeleteTopicInNameSrv, request, nil)
- return a.cli.InvokeSync(ctx, nameSrvAddr, cmd, 5*time.Second)
-}
-
-// DeleteTopic delete topic in both broker and nameserver.
-func (a *admin) DeleteTopic(ctx context.Context, opts ...OptionDelete) error {
- cfg := defaultTopicConfigDelete()
- for _, apply := range opts {
- apply(&cfg)
- }
- //delete topic in broker
- if cfg.BrokerAddr == "" {
- a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- cfg.BrokerAddr = a.cli.GetNameSrv().FindBrokerAddrByTopic(cfg.Topic)
- }
-
- if _, err := a.deleteTopicInBroker(ctx, cfg.Topic, cfg.BrokerAddr); err != nil {
- rlog.Error("delete topic in broker error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- return err
- }
-
- //delete topic in nameserver
- if len(cfg.NameSrvAddr) == 0 {
- a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- cfg.NameSrvAddr = a.cli.GetNameSrv().AddrList()
- _, _, err := a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- if err != nil {
- rlog.Error("delete topic in nameserver error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyUnderlayError: err,
- })
- }
- cfg.NameSrvAddr = a.cli.GetNameSrv().AddrList()
- }
-
- for _, nameSrvAddr := range cfg.NameSrvAddr {
- if _, err := a.deleteTopicInNameServer(ctx, cfg.Topic, nameSrvAddr); err != nil {
- rlog.Error("delete topic in nameserver error", map[string]interface{}{
- "nameServer": nameSrvAddr,
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyUnderlayError: err,
- })
- return err
- }
- }
- rlog.Info("delete topic success", map[string]interface{}{
- "nameServer": cfg.NameSrvAddr,
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- })
- return nil
-}
-
-func (a *admin) Close() error {
- a.closeOnce.Do(func() {
- a.cli.Shutdown()
- })
- return nil
-}
diff --git a/admin/option.go b/admin/option.go
deleted file mode 100644
index d5a648e..0000000
--- a/admin/option.go
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package admin
-
-func defaultTopicConfigCreate() TopicConfigCreate {
- opts := TopicConfigCreate{
- DefaultTopic: "defaultTopic",
- ReadQueueNums: 8,
- WriteQueueNums: 8,
- Perm: 6,
- TopicFilterType: "SINGLE_TAG",
- TopicSysFlag: 0,
- Order: false,
- }
- return opts
-}
-
-type TopicConfigCreate struct {
- Topic string
- BrokerAddr string
- DefaultTopic string
- ReadQueueNums int
- WriteQueueNums int
- Perm int
- TopicFilterType string
- TopicSysFlag int
- Order bool
-}
-
-type OptionCreate func(*TopicConfigCreate)
-
-func WithTopicCreate(Topic string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Topic = Topic
- }
-}
-
-func WithBrokerAddrCreate(BrokerAddr string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.BrokerAddr = BrokerAddr
- }
-}
-
-func WithReadQueueNums(ReadQueueNums int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.ReadQueueNums = ReadQueueNums
- }
-}
-
-func WithWriteQueueNums(WriteQueueNums int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.WriteQueueNums = WriteQueueNums
- }
-}
-
-func WithPerm(Perm int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Perm = Perm
- }
-}
-
-func WithTopicFilterType(TopicFilterType string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.TopicFilterType = TopicFilterType
- }
-}
-
-func WithTopicSysFlag(TopicSysFlag int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.TopicSysFlag = TopicSysFlag
- }
-}
-
-func WithOrder(Order bool) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Order = Order
- }
-}
-
-func defaultTopicConfigDelete() TopicConfigDelete {
- opts := TopicConfigDelete{}
- return opts
-}
-
-type TopicConfigDelete struct {
- Topic string
- ClusterName string
- NameSrvAddr []string
- BrokerAddr string
-}
-
-type OptionDelete func(*TopicConfigDelete)
-
-func WithTopicDelete(Topic string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.Topic = Topic
- }
-}
-
-func WithBrokerAddrDelete(BrokerAddr string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.BrokerAddr = BrokerAddr
- }
-}
-
-func WithClusterName(ClusterName string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.ClusterName = ClusterName
- }
-}
-
-func WithNameSrvAddr(NameSrvAddr []string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.NameSrvAddr = NameSrvAddr
- }
-}
diff --git a/api.go b/api.go
deleted file mode 100644
index 31f58d5..0000000
--- a/api.go
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package rocketmq
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/errors"
-
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-type Producer interface {
- Start() error
- Shutdown() error
- SendSync(ctx context.Context, mq ...*primitive.Message) (*primitive.SendResult, error)
- SendAsync(ctx context.Context, mq func(ctx context.Context, result *primitive.SendResult, err error),
- msg ...*primitive.Message) error
- SendOneWay(ctx context.Context, mq ...*primitive.Message) error
-}
-
-func NewProducer(opts ...producer.Option) (Producer, error) {
- return producer.NewDefaultProducer(opts...)
-}
-
-type TransactionProducer interface {
- Start() error
- Shutdown() error
- SendMessageInTransaction(ctx context.Context, mq *primitive.Message) (*primitive.TransactionSendResult, error)
-}
-
-func NewTransactionProducer(listener primitive.TransactionListener, opts ...producer.Option) (TransactionProducer, error) {
- return producer.NewTransactionProducer(listener, opts...)
-}
-
-type PushConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be sync to broker before process exit
- Shutdown() error
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (consumer.ConsumeResult, error)) error
-
- // Unsubscribe a topic
- Unsubscribe(topic string) error
-}
-
-func NewPushConsumer(opts ...consumer.Option) (PushConsumer, error) {
- return consumer.NewPushConsumer(opts...)
-}
-
-type PullConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be commit to broker before process exit
- Shutdown() error
-
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector) error
-
- // Unsubscribe a topic
- Unsubscribe(topic string) error
-
- // MessageQueues get MessageQueue list about for a given topic. This method will issue a remote call to the server
- // if it does not already have any MessageQueue about the given topic.
- MessageQueues(topic string) []primitive.MessageQueue
-
- // Pull message for the topic specified. It is an error to not have subscribed to any topics before pull for message
- //
- // Specified numbers of messages is returned if message greater that numbers, and the offset will auto forward.
- // It means that if you meeting messages consuming failed, you should process failed messages by yourself.
- Pull(ctx context.Context, topic string, numbers int) (*primitive.PullResult, error)
-
- // Pull message for the topic specified from a specified MessageQueue and offset. It is an error to not have
- // subscribed to any topics before pull for message. the method will not affect the offset recorded
- //
- // Specified numbers of messages is returned.
- PullFrom(ctx context.Context, mq primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error)
-
- // Lookup offset for the given message queue by timestamp. The returned offset for the message queue is the
- // earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message
- // queue.
- //
- // Timestamp must be millisecond level, if you want to lookup the earliest offset of the mq, you could set the
- // timestamp 0, and if you want to the latest offset the mq, you could set the timestamp math.MaxInt64.
- Lookup(ctx context.Context, mq primitive.MessageQueue, timestamp int64) (int64, error)
-
- // Commit the offset of specified mqs to broker, if auto-commit is disable, you must commit the offset manually.
- Commit(ctx context.Context, mqs ...primitive.MessageQueue) (int64, error)
-
- // CommittedOffset return the offset of specified Message
- CommittedOffset(mq primitive.MessageQueue) (int64, error)
-
- // Seek set offset of the mq, if you wanna re-consuming your message form one position, the method may help you.
- // if you want re-consuming from one time, you cloud Lookup() then seek it.
- Seek(mq primitive.MessageQueue, offset int64) error
-
- // Pause consuming for specified MessageQueues, after pause, client will not fetch any message from the specified
- // message queues
- //
- // Note that this method does not affect message queue subscription. In particular, it does not cause a group
- // rebalance.
- //
- // if a MessageQueue belong a topic that has not been subscribed, an error will be returned
- //Pause(mqs ...primitive.MessageQueue) error
-
- // Resume specified message queues which have been paused with Pause, if a MessageQueue that not paused,
- // it will be ignored. if not subscribed, an error will be returned
- //Resume(mqs ...primitive.MessageQueue) error
-}
-
-// The PullConsumer has not implemented completely, if you want have an experience of PullConsumer, you could use
-// consumer.NewPullConsumer(...), but it may changed in the future.
-//
-// The PullConsumer will be supported in next release
-func NewPullConsumer(opts ...consumer.Option) (PullConsumer, error) {
- return nil, errors.ErrPullConsumer
-}
diff --git a/benchmark/consumer.go b/benchmark/consumer.go
deleted file mode 100644
index 907a1e7..0000000
--- a/benchmark/consumer.go
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "context"
- "flag"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "sync"
- "sync/atomic"
- "syscall"
- "time"
-)
-
-type statiBenchmarkConsumerSnapshot struct {
- receiveMessageTotal int64
- born2ConsumerTotalRT int64
- store2ConsumerTotalRT int64
- born2ConsumerMaxRT int64
- store2ConsumerMaxRT int64
- createdAt time.Time
- next *statiBenchmarkConsumerSnapshot
-}
-
-type consumeSnapshots struct {
- sync.RWMutex
- head, tail, cur *statiBenchmarkConsumerSnapshot
- len int
-}
-
-func (s *consumeSnapshots) takeSnapshot() {
- b := s.cur
- sn := new(statiBenchmarkConsumerSnapshot)
- sn.receiveMessageTotal = atomic.LoadInt64(&b.receiveMessageTotal)
- sn.born2ConsumerMaxRT = atomic.LoadInt64(&b.born2ConsumerMaxRT)
- sn.born2ConsumerTotalRT = atomic.LoadInt64(&b.born2ConsumerTotalRT)
- sn.store2ConsumerMaxRT = atomic.LoadInt64(&b.store2ConsumerMaxRT)
- sn.store2ConsumerTotalRT = atomic.LoadInt64(&b.store2ConsumerTotalRT)
- sn.createdAt = time.Now()
-
- s.Lock()
- if s.tail != nil {
- s.tail.next = sn
- }
- s.tail = sn
- if s.head == nil {
- s.head = s.tail
- }
-
- s.len++
- if s.len > 10 {
- s.head = s.head.next
- s.len--
- }
- s.Unlock()
-}
-
-func (s *consumeSnapshots) printStati() {
- s.RLock()
- if s.len < 10 {
- s.RUnlock()
- return
- }
-
- f, l := s.head, s.tail
- respSucCount := float64(l.receiveMessageTotal - f.receiveMessageTotal)
- consumeTps := respSucCount / l.createdAt.Sub(f.createdAt).Seconds()
- avgB2CRT := float64(l.born2ConsumerTotalRT-f.born2ConsumerTotalRT) / respSucCount
- avgS2CRT := float64(l.store2ConsumerTotalRT-f.store2ConsumerTotalRT) / respSucCount
- s.RUnlock()
-
- rlog.Info("Benchmark Consumer Snapshot", map[string]interface{}{
- "consumeTPS": int64(consumeTps),
- "average(B2C)RT": avgB2CRT,
- "average(S2C)RT": avgS2CRT,
- "max(B2C)RT": l.born2ConsumerMaxRT,
- "max(S2C)RT": l.store2ConsumerMaxRT,
- })
-}
-
-type consumerBenchmark struct {
- topic string
- groupPrefix string
- nameSrv string
- isPrefixEnable bool
- filterType string
- expression string
- testMinutes int
- instanceCount int
-
- flags *flag.FlagSet
-
- groupID string
-}
-
-func init() {
- c := &consumerBenchmark{}
- flags := flag.NewFlagSet("consumer", flag.ExitOnError)
- c.flags = flags
-
- flags.StringVar(&c.topic, "t", "BenchmarkTest", "topic")
- flags.StringVar(&c.groupPrefix, "g", "benchmark_consumer", "group prefix")
- flags.StringVar(&c.nameSrv, "n", "", "namesrv address list, separated by comma")
- flags.BoolVar(&c.isPrefixEnable, "p", true, "group prefix is enable")
- flags.StringVar(&c.filterType, "f", "", "filter type,options:TAG|SQL92, or empty")
- flags.StringVar(&c.expression, "e", "*", "expression")
- flags.IntVar(&c.testMinutes, "m", 10, "test minutes")
- flags.IntVar(&c.instanceCount, "i", 1, "instance count")
-
- registerCommand("consumer", c)
-}
-
-func (bc *consumerBenchmark) consumeMsg(stati *statiBenchmarkConsumerSnapshot, exit chan struct{}) {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName(bc.groupID),
- consumer.WithNameServer([]string{bc.nameSrv}),
- )
- if err != nil {
- panic("new push consumer error:" + err.Error())
- }
-
- selector := consumer.MessageSelector{}
- err = c.Subscribe(bc.topic, selector, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- for _, msg := range msgs {
- atomic.AddInt64(&stati.receiveMessageTotal, 1)
- now := time.Now().UnixNano() / int64(time.Millisecond)
- b2cRT := now - msg.BornTimestamp
- atomic.AddInt64(&stati.born2ConsumerTotalRT, b2cRT)
- s2cRT := now - msg.StoreTimestamp
- atomic.AddInt64(&stati.store2ConsumerTotalRT, s2cRT)
-
- for {
- old := atomic.LoadInt64(&stati.born2ConsumerMaxRT)
- if old >= b2cRT || atomic.CompareAndSwapInt64(&stati.born2ConsumerMaxRT, old, b2cRT) {
- break
- }
- }
-
- for {
- old := atomic.LoadInt64(&stati.store2ConsumerMaxRT)
- if old >= s2cRT || atomic.CompareAndSwapInt64(&stati.store2ConsumerMaxRT, old, s2cRT) {
- break
- }
- }
- }
- return consumer.ConsumeSuccess, nil
- })
-
- rlog.Info("Test Start", nil)
- c.Start()
- select {
- case <-exit:
- c.Shutdown()
- return
- }
-}
-
-func (bc *consumerBenchmark) run(args []string) {
- bc.flags.Parse(args)
- if bc.topic == "" {
- rlog.Error("Empty Topic", nil)
- bc.usage()
- return
- }
-
- if bc.groupPrefix == "" {
- rlog.Error("Empty Group Prefix", nil)
- bc.usage()
- return
- }
-
- if bc.nameSrv == "" {
- rlog.Error("Empty Nameserver", nil)
- bc.usage()
- return
- }
-
- if bc.testMinutes <= 0 {
- rlog.Error("Test Time Must Be Positive Integer", nil)
- bc.usage()
- return
- }
-
- if bc.instanceCount <= 0 {
- rlog.Error("Thread Count Must Be Positive Integer", nil)
- bc.usage()
- return
- }
-
- bc.groupID = bc.groupPrefix
- if bc.isPrefixEnable {
- bc.groupID += fmt.Sprintf("_%d", time.Now().UnixNano()/int64(time.Millisecond)%100)
- }
-
- stati := statiBenchmarkConsumerSnapshot{}
- snapshots := consumeSnapshots{cur: &stati}
- exitChan := make(chan struct{})
-
- wg := sync.WaitGroup{}
-
- wg.Add(1)
- go func() {
- bc.consumeMsg(&stati, exitChan)
- wg.Done()
- }()
-
- // snapshot
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second)
- for {
- select {
- case <-ticker.C:
- snapshots.takeSnapshot()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- // print statistic
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second * 10)
- for {
- select {
- case <-ticker.C:
- snapshots.printStati()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- select {
- case <-time.Tick(time.Minute * time.Duration(bc.testMinutes)):
- case <-signalChan:
- }
-
- close(exitChan)
- wg.Wait()
- snapshots.takeSnapshot()
- snapshots.printStati()
- rlog.Info("Test Done", nil)
-}
-
-func (bc *consumerBenchmark) usage() {
- bc.flags.Usage()
-}
diff --git a/benchmark/main.go b/benchmark/main.go
deleted file mode 100644
index 79eca8f..0000000
--- a/benchmark/main.go
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
-)
-
-type command interface {
- usage()
- run(args []string)
-}
-
-var (
- cmds = map[string]command{}
-)
-
-func registerCommand(name string, cmd command) {
- if cmd == nil {
- panic("empty command")
- }
-
- _, ok := cmds[name]
- if ok {
- panic(fmt.Sprintf("%s command existed", name))
- }
-
- cmds[name] = cmd
-}
-
-func usage() {
- rlog.Info("Command", map[string]interface{}{
- "name": os.Args[0],
- })
- for _, cmd := range cmds {
- cmd.usage()
- }
-}
-
-// go run *.go [command name] [command args]
-func main() {
- if len(os.Args) < 2 {
- rlog.Error("Lack Command Name", nil)
- usage()
- return
- }
-
- name := os.Args[1]
- cmd, ok := cmds[name]
- if !ok {
- rlog.Error("Command Isn't Supported", map[string]interface{}{
- "command": name,
- })
- usage()
- return
- }
-
- cmd.run(os.Args[2:])
-}
diff --git a/benchmark/message.go b/benchmark/message.go
deleted file mode 100644
index d5690fe..0000000
--- a/benchmark/message.go
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "strings"
-
-var (
- longText = ""
- longTextLen = 0
-)
-
-func init() {
- longText = strings.Repeat("0123456789", 100)
- longTextLen = len(longText)
-}
-
-func buildMsg(size int) string {
- return longText[:size]
-}
diff --git a/benchmark/producer.go b/benchmark/producer.go
deleted file mode 100644
index 7516352..0000000
--- a/benchmark/producer.go
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "context"
- "flag"
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "sync"
- "sync/atomic"
- "syscall"
- "time"
-)
-
-type statiBenchmarkProducerSnapshot struct {
- sendRequestSuccessCount int64
- sendRequestFailedCount int64
- receiveResponseSuccessCount int64
- receiveResponseFailedCount int64
- sendMessageSuccessTimeTotal int64
- sendMessageMaxRT int64
- createdAt time.Time
- next *statiBenchmarkProducerSnapshot
-}
-
-type produceSnapshots struct {
- sync.RWMutex
- head, tail, cur *statiBenchmarkProducerSnapshot
- len int
-}
-
-func (s *produceSnapshots) takeSnapshot() {
- b := s.cur
- sn := new(statiBenchmarkProducerSnapshot)
- sn.sendRequestSuccessCount = atomic.LoadInt64(&b.sendRequestSuccessCount)
- sn.sendRequestFailedCount = atomic.LoadInt64(&b.sendRequestFailedCount)
- sn.receiveResponseSuccessCount = atomic.LoadInt64(&b.receiveResponseSuccessCount)
- sn.receiveResponseFailedCount = atomic.LoadInt64(&b.receiveResponseFailedCount)
- sn.sendMessageSuccessTimeTotal = atomic.LoadInt64(&b.sendMessageSuccessTimeTotal)
- sn.sendMessageMaxRT = atomic.LoadInt64(&b.sendMessageMaxRT)
- sn.createdAt = time.Now()
-
- s.Lock()
- if s.tail != nil {
- s.tail.next = sn
- }
- s.tail = sn
- if s.head == nil {
- s.head = s.tail
- }
-
- s.len++
- if s.len > 10 {
- s.head = s.head.next
- s.len--
- }
- s.Unlock()
-}
-
-func (s *produceSnapshots) printStati() {
- s.RLock()
- if s.len < 10 {
- s.RUnlock()
- return
- }
-
- f, l := s.head, s.tail
- respSucCount := float64(l.receiveResponseSuccessCount - f.receiveResponseSuccessCount)
- sendTps := respSucCount / l.createdAt.Sub(f.createdAt).Seconds()
- avgRT := float64(l.sendMessageSuccessTimeTotal-f.sendMessageSuccessTimeTotal) / respSucCount
- maxRT := atomic.LoadInt64(&s.cur.sendMessageMaxRT)
- s.RUnlock()
-
- rlog.Info("Benchmark Producer Snapshot", map[string]interface{}{
- "sendTps": int64(sendTps),
- "maxRt": maxRT,
- "averageRt": avgRT,
- "sendFailed": l.sendRequestFailedCount,
- "responseFailed": l.receiveResponseFailedCount,
- "total": l.receiveResponseSuccessCount,
- })
-}
-
-type producerBenchmark struct {
- topic string
- nameSrv string
- groupID string
- instanceCount int
- testMinutes int
- bodySize int
-
- flags *flag.FlagSet
-}
-
-func init() {
- p := &producerBenchmark{}
- flags := flag.NewFlagSet("producer", flag.ExitOnError)
- p.flags = flags
-
- flags.StringVar(&p.topic, "t", "", "topic name")
- flags.StringVar(&p.nameSrv, "n", "", "nameserver address")
- flags.StringVar(&p.groupID, "g", "", "group id")
- flags.IntVar(&p.instanceCount, "i", 1, "instance count")
- flags.IntVar(&p.testMinutes, "m", 10, "test minutes")
- flags.IntVar(&p.bodySize, "s", 32, "body size")
-
- registerCommand("producer", p)
-}
-
-func (bp *producerBenchmark) produceMsg(stati *statiBenchmarkProducerSnapshot, exit chan struct{}) {
- p, err := rocketmq.NewProducer(
- producer.WithNameServer([]string{bp.nameSrv}),
- producer.WithRetry(2),
- )
-
- if err != nil {
- rlog.Error("New Producer Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return
- }
-
- err = p.Start()
-
- defer p.Shutdown()
-
- topic, tag := bp.topic, "benchmark-producer"
- msgStr := buildMsg(bp.bodySize)
-
-AGAIN:
- select {
- case <-exit:
- return
- default:
- }
-
- now := time.Now()
- r, err := p.SendSync(context.Background(), primitive.NewMessage(topic, []byte(msgStr)))
-
- if err != nil {
- rlog.Error("Send Message Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- goto AGAIN
- }
-
- if r.Status == primitive.SendOK {
- atomic.AddInt64(&stati.receiveResponseSuccessCount, 1)
- atomic.AddInt64(&stati.sendRequestSuccessCount, 1)
- currentRT := int64(time.Since(now) / time.Millisecond)
- atomic.AddInt64(&stati.sendMessageSuccessTimeTotal, currentRT)
- prevRT := atomic.LoadInt64(&stati.sendMessageMaxRT)
- for currentRT > prevRT {
- if atomic.CompareAndSwapInt64(&stati.sendMessageMaxRT, prevRT, currentRT) {
- break
- }
- prevRT = atomic.LoadInt64(&stati.sendMessageMaxRT)
- }
- goto AGAIN
- }
- rlog.Error("Send Message Error", map[string]interface{}{
- "topic": topic,
- "tag": tag,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- goto AGAIN
-}
-
-func (bp *producerBenchmark) run(args []string) {
- bp.flags.Parse(args)
-
- if bp.topic == "" {
- rlog.Error("Empty Topic", nil)
- bp.flags.Usage()
- return
- }
-
- if bp.groupID == "" {
- rlog.Error("Empty Group Id", nil)
- bp.flags.Usage()
- return
- }
-
- if bp.nameSrv == "" {
- rlog.Error("Empty Nameserver", nil)
- bp.flags.Usage()
- return
- }
- if bp.instanceCount <= 0 {
- rlog.Error("Instance Count Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
- if bp.testMinutes <= 0 {
- rlog.Error("Test Time Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
- if bp.bodySize <= 0 {
- rlog.Error("Body Size Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
-
- stati := statiBenchmarkProducerSnapshot{}
- snapshots := produceSnapshots{cur: &stati}
- exitChan := make(chan struct{})
- wg := sync.WaitGroup{}
-
- for i := 0; i < bp.instanceCount; i++ {
- i := i
- go func() {
- wg.Add(1)
- bp.produceMsg(&stati, exitChan)
- rlog.Info("Producer Done and Exit", map[string]interface{}{
- "id": i,
- })
- wg.Done()
- }()
- }
-
- // snapshot
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second)
- for {
- select {
- case <-ticker.C:
- snapshots.takeSnapshot()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- // print statistic
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second * 10)
- for {
- select {
- case <-ticker.C:
- snapshots.printStati()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- select {
- case <-time.Tick(time.Minute * time.Duration(bp.testMinutes)):
- case <-signalChan:
- }
-
- close(exitChan)
- wg.Wait()
- snapshots.takeSnapshot()
- snapshots.printStati()
- rlog.Info("Test Done", nil)
-}
-
-func (bp *producerBenchmark) usage() {
- bp.flags.Usage()
-}
diff --git a/benchmark/stable.go b/benchmark/stable.go
deleted file mode 100644
index 2659bc5..0000000
--- a/benchmark/stable.go
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "flag"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "syscall"
- "time"
-)
-
-type stableTest struct {
- nameSrv string
- topic string
- groupID string
- opIntervalSec int
- testMin int
-
- op func()
-
- flags *flag.FlagSet
-}
-
-func (st *stableTest) buildFlags(name string) {
- flags := flag.NewFlagSet(name, flag.ExitOnError)
- flags.StringVar(&st.topic, "t", "stable-test", "topic name")
- flags.StringVar(&st.nameSrv, "n", "", "nameserver address")
- flags.StringVar(&st.groupID, "g", "stable-test", "group id")
- flags.IntVar(&st.testMin, "m", 10, "test minutes")
- flags.IntVar(&st.opIntervalSec, "s", 1, "operation interval[produce/consume]")
-
- st.flags = flags
-}
-
-func (st *stableTest) checkFlag() error {
- if st.topic == "" {
- return errors.ErrEmptyTopic
- }
-
- if st.nameSrv == "" {
- return errors.ErrEmptyNameSrv
- }
-
- if st.groupID == "" {
- return errors.ErrEmptyGroupID
- }
-
- if st.testMin <= 0 {
- return errors.ErrTestMin
- }
-
- if st.opIntervalSec <= 0 {
- return errors.ErrOperationInterval
- }
-
- return nil
-}
-
-func (st *stableTest) run() {
- opTicker := time.NewTicker(time.Duration(st.opIntervalSec) * time.Second)
- closeChan := time.Tick(time.Duration(st.testMin) * time.Minute)
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- for {
- select {
- case <-signalChan:
- opTicker.Stop()
- rlog.Info("Test Done", nil)
- return
- case <-closeChan:
- opTicker.Stop()
- rlog.Info("Test Done", nil)
- return
- case <-opTicker.C:
- st.op()
- }
- }
-}
-
-type stableTestProducer struct {
- *stableTest
- bodySize int
-
- //p rocketmq.Producer
-}
-
-func (stp *stableTestProducer) buildFlags(name string) {
- stp.stableTest.buildFlags(name)
- stp.flags.IntVar(&stp.bodySize, "b", 32, "body size")
-}
-
-func (stp *stableTestProducer) checkFlag() error {
- err := stp.stableTest.checkFlag()
- if err != nil {
- return err
- }
- if stp.bodySize <= 0 {
- return errors.ErrMessageBody
- }
-
- return nil
-}
-
-func (stp *stableTestProducer) usage() {
- stp.flags.Usage()
-}
-
-func (stp *stableTestProducer) run(args []string) {
- err := stp.flags.Parse(args)
- if err != nil {
- rlog.Info("Parse Args Error", map[string]interface{}{
- "args": args,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stp.usage()
- return
- }
-
- err = stp.checkFlag()
- if err != nil {
- rlog.Error("Check Flag Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stp.usage()
- return
- }
-
- //p, err := rocketmq.NewProducer(&rocketmq.ProducerConfig{
- // ClientConfig: rocketmq.ClientConfig{GroupID: stp.groupID, NameServer: stp.nameSrv},
- //})
- //if err != nil {
- // fmt.Printf("new consumer error:%s\n", err)
- // return
- //}
- //
- //err = p.Start()
- //if err != nil {
- // fmt.Printf("start consumer error:%s\n", err)
- // return
- //}
- //defer p.Shutdown()
- //
- //stp.p = p
- stp.stableTest.run()
-}
-
-func (stp *stableTestProducer) sendMessage() {
- //r, err := stp.p.SendMessageSync(&rocketmq.Message{Topic: stp.topic, Body: buildMsg(stp.bodySize)})
- //if err == nil {
- // fmt.Printf("send result:%+v\n", r)
- // return
- //}
- //fmt.Printf("send message error:%s", err)
-}
-
-type stableTestConsumer struct {
- *stableTest
- expression string
-
- //c rocketmq.PullConsumer
- offsets map[int]int64
-}
-
-func (stc *stableTestConsumer) buildFlags(name string) {
- stc.stableTest.buildFlags(name)
- stc.flags.StringVar(&stc.expression, "e", "*", "expression")
-}
-
-func (stc *stableTestConsumer) checkFlag() error {
- err := stc.stableTest.checkFlag()
- if err != nil {
- return err
- }
-
- if stc.expression == "" {
- return errors.ErrEmptyExpression
- }
- return nil
-}
-
-func (stc *stableTestConsumer) usage() {
- stc.flags.Usage()
-}
-
-func (stc *stableTestConsumer) run(args []string) {
- err := stc.flags.Parse(args)
- if err != nil {
- rlog.Error("Parse Args Error", map[string]interface{}{
- "args": args,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stc.usage()
- return
- }
-
- err = stc.checkFlag()
- if err != nil {
- rlog.Error("Check Flag Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stc.usage()
- return
- }
- //
- //c, err := rocketmq.NewPullConsumer(&rocketmq.PullConsumerConfig{
- // ClientConfig: rocketmq.ClientConfig{GroupID: stc.groupID, NameServer: stc.nameSrv},
- //})
- //if err != nil {
- // fmt.Printf("new pull consumer error:%s\n", err)
- // return
- //}
- //
- //err = c.Start()
- //if err != nil {
- // fmt.Printf("start consumer error:%s\n", err)
- // return
- //}
- //defer c.Shutdown()
- //
- //stc.c = c
- stc.stableTest.run()
-}
-
-func (stc *stableTestConsumer) pullMessage() {
- //mqs := stc.c.FetchSubscriptionMessageQueues(stc.topic)
- //
- //for _, mq := range mqs {
- // offset := stc.offsets[mq.ID]
- // pr := stc.c.Pull(mq, stc.expression, offset, 32)
- //fmt.Printf("pull from %s, offset:%d, count:%+v\n", mq.String(), offset, len(pr.Messages))
- //
- //switch pr.Status {
- //case rocketmq.PullNoNewMsg:
- // stc.offsets[mq.ID] = 0 // pull from the begin
- //case rocketmq.PullFound:
- // fallthrough
- //case rocketmq.PullNoMatchedMsg:
- // fallthrough
- //case rocketmq.PullOffsetIllegal:
- // stc.offsets[mq.ID] = pr.NextBeginOffset
- //case rocketmq.PullBrokerTimeout:
- // fmt.Println("broker timeout occur")
- //}
- //}
-}
-
-func init() {
- // consumer
- name := "stableTestProducer"
- p := &stableTestProducer{stableTest: &stableTest{}}
- p.buildFlags(name)
- p.op = p.sendMessage
- registerCommand(name, p)
-
- // consumer
- name = "stableTestConsumer"
- c := &stableTestConsumer{stableTest: &stableTest{}, offsets: map[int]int64{}}
- c.buildFlags(name)
- c.op = c.pullMessage
- registerCommand(name, c)
-}
diff --git a/config.go b/config.go
deleted file mode 100644
index 8f1592c..0000000
--- a/config.go
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-package rocketmq
diff --git a/consumer/consumer.go b/consumer/consumer.go
deleted file mode 100644
index 8056b22..0000000
--- a/consumer/consumer.go
+++ /dev/null
@@ -1,1063 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "sort"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- jsoniter "github.com/json-iterator/go"
-
- "github.com/tidwall/gjson"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- // Delay some time when exception error
- _PullDelayTimeWhenError = 3 * time.Second
-
- // Flow control interval
- _PullDelayTimeWhenFlowControl = 50 * time.Millisecond
-
- // Delay some time when suspend pull service
- _PullDelayTimeWhenSuspend = 30 * time.Second
-
- // Long polling mode, the Consumer connection max suspend time
- _BrokerSuspendMaxTime = 20 * time.Second
-
- // Long polling mode, the Consumer connection timeout (must greater than _BrokerSuspendMaxTime)
- _ConsumerTimeoutWhenSuspend = 30 * time.Second
-
- // Offset persistent interval for consumer
- _PersistConsumerOffsetInterval = 5 * time.Second
-)
-
-type ConsumeType string
-
-const (
- _PullConsume = ConsumeType("CONSUME_ACTIVELY")
- _PushConsume = ConsumeType("CONSUME_PASSIVELY")
-
- _SubAll = "*"
-)
-
-// Message model defines the way how messages are delivered to each consumer clients.
-// </p>
-//
-// RocketMQ supports two message models: clustering and broadcasting. If clustering is set, consumer clients with
-// the same {@link #ConsumerGroup} would only consume shards of the messages subscribed, which achieves load
-// balances; Conversely, if the broadcasting is set, each consumer client will consume all subscribed messages
-// separately.
-// </p>
-//
-// This field defaults to clustering.
-type MessageModel int
-
-const (
- BroadCasting MessageModel = iota
- Clustering
-)
-
-func (mode MessageModel) String() string {
- switch mode {
- case BroadCasting:
- return "BroadCasting"
- case Clustering:
- return "Clustering"
- default:
- return "Unknown"
- }
-}
-
-// Consuming point on consumer booting.
-// </p>
-//
-// There are three consuming points:
-// <ul>
-// <li>
-// <code>CONSUME_FROM_LAST_OFFSET</code>: consumer clients pick up where it stopped previously.
-// If it were a newly booting up consumer client, according aging of the consumer group, there are two
-// cases:
-// <ol>
-// <li>
-// if the consumer group is created so recently that the earliest message being subscribed has yet
-// expired, which means the consumer group represents a lately launched business, consuming will
-// start from the very beginning;
-// </li>
-// <li>
-// if the earliest message being subscribed has expired, consuming will start from the latest
-// messages, meaning messages born prior to the booting timestamp would be ignored.
-// </li>
-// </ol>
-// </li>
-// <li>
-// <code>CONSUME_FROM_FIRST_OFFSET</code>: Consumer client will start from earliest messages available.
-// </li>
-// <li>
-// <code>CONSUME_FROM_TIMESTAMP</code>: Consumer client will start from specified timestamp, which means
-// messages born prior to {@link #consumeTimestamp} will be ignored
-// </li>
-// </ul>
-type ConsumeFromWhere int
-
-const (
- ConsumeFromLastOffset ConsumeFromWhere = iota
- ConsumeFromFirstOffset
- ConsumeFromTimestamp
-)
-
-type ExpressionType string
-
-const (
- /**
- * <ul>
- * Keywords:
- * <li>{@code AND, OR, NOT, BETWEEN, IN, TRUE, FALSE, IS, NULL}</li>
- * </ul>
- * <p/>
- * <ul>
- * Data type:
- * <li>Boolean, like: TRUE, FALSE</li>
- * <li>String, like: 'abc'</li>
- * <li>Decimal, like: 123</li>
- * <li>Float number, like: 3.1415</li>
- * </ul>
- * <p/>
- * <ul>
- * Grammar:
- * <li>{@code AND, OR}</li>
- * <li>{@code >, >=, <, <=, =}</li>
- * <li>{@code BETWEEN A AND B}, equals to {@code >=A AND <=B}</li>
- * <li>{@code NOT BETWEEN A AND B}, equals to {@code >B OR <A}</li>
- * <li>{@code IN ('a', 'b')}, equals to {@code ='a' OR ='b'}, this operation only support String type.</li>
- * <li>{@code IS NULL}, {@code IS NOT NULL}, check parameter whether is null, or not.</li>
- * <li>{@code =TRUE}, {@code =FALSE}, check parameter whether is true, or false.</li>
- * </ul>
- * <p/>
- * <p>
- * Example:
- * (a > 10 AND a < 100) OR (b IS NOT NULL AND b=TRUE)
- * </p>
- */
- SQL92 = ExpressionType("SQL92")
-
- /**
- * Only support or operation such as
- * "tag1 || tag2 || tag3", <br>
- * If null or * expression, meaning subscribe all.
- */
- TAG = ExpressionType("TAG")
-)
-
-func IsTagType(exp string) bool {
- if exp == "" || exp == "TAG" {
- return true
- }
- return false
-}
-
-type MessageSelector struct {
- Type ExpressionType
- Expression string
-}
-
-type ConsumeResult int
-
-const (
- ConsumeSuccess ConsumeResult = iota
- ConsumeRetryLater
- Commit
- Rollback
- SuspendCurrentQueueAMoment
-)
-
-type ConsumeResultHolder struct {
- ConsumeResult
-}
-
-type ConsumerReturn int
-
-const (
- SuccessReturn ConsumerReturn = iota
- ExceptionReturn
- NullReturn
- TimeoutReturn
- FailedReturn
-)
-
-type PullRequest struct {
- consumerGroup string
- mq *primitive.MessageQueue
- pq *processQueue
- nextOffset int64
- lockedFirst bool
-}
-
-func (pr *PullRequest) String() string {
- return fmt.Sprintf("[ConsumerGroup: %s, Topic: %s, MessageQueue: %d]",
- pr.consumerGroup, pr.mq.Topic, pr.mq.QueueId)
-}
-
-type defaultConsumer struct {
- /**
- * Consumers of the same role is required to have exactly same subscriptions and consumerGroup to correctly achieve
- * load balance. It's required and needs to be globally unique.
- * </p>
- *
- * See <a href="http://rocketmq.apache.org/docs/core-concept/">here</a> for further discussion.
- */
- consumerGroup string
- model MessageModel
- allocate func(string, string, []*primitive.MessageQueue, []string) []*primitive.MessageQueue
- unitMode bool
- consumeOrderly bool
- fromWhere ConsumeFromWhere
- consumerStartTimestamp int64
-
- cType ConsumeType
- client internal.RMQClient
- mqChanged func(topic string, mqAll, mqDivided []*primitive.MessageQueue)
- state int32
- pause bool
- once sync.Once
- option consumerOptions
- // key: primitive.MessageQueue
- // value: *processQueue
- processQueueTable sync.Map
-
- // key: topic(string)
- // value: map[int]*primitive.MessageQueue
- topicSubscribeInfoTable sync.Map
-
- // key: topic
- // value: *SubscriptionData
- subscriptionDataTable sync.Map
- storage OffsetStore
- // chan for push consumer
- prCh chan PullRequest
-
- pullFromWhichNodeTable sync.Map
-
- stat *StatsManager
-}
-
-func (dc *defaultConsumer) start() error {
- if dc.model == Clustering {
- // set retry topic
- retryTopic := internal.GetRetryTopic(dc.consumerGroup)
- sub := buildSubscriptionData(retryTopic, MessageSelector{TAG, _SubAll})
- dc.subscriptionDataTable.Store(retryTopic, sub)
- }
-
- if dc.model == Clustering {
- dc.option.ChangeInstanceNameToPID()
- dc.storage = NewRemoteOffsetStore(dc.consumerGroup, dc.client, dc.client.GetNameSrv())
- } else {
- dc.storage = NewLocalFileOffsetStore(dc.consumerGroup, dc.client.ClientID())
- }
-
- dc.client.Start()
- atomic.StoreInt32(&dc.state, int32(internal.StateRunning))
- dc.consumerStartTimestamp = time.Now().UnixNano() / int64(time.Millisecond)
- dc.stat = NewStatsManager()
- return nil
-}
-
-func (dc *defaultConsumer) shutdown() error {
- atomic.StoreInt32(&dc.state, int32(internal.StateShutdown))
-
- mqs := make([]*primitive.MessageQueue, 0)
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- k := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- pq.WithDropped(true)
- // close msg channel using RWMutex to make sure no data was writing
- pq.mutex.Lock()
- close(pq.msgCh)
- pq.mutex.Unlock()
- mqs = append(mqs, &k)
- return true
- })
- dc.stat.ShutDownStat()
- dc.storage.persist(mqs)
- dc.client.Shutdown()
- return nil
-}
-
-func (dc *defaultConsumer) persistConsumerOffset() error {
- err := dc.makeSureStateOK()
- if err != nil {
- return err
- }
- mqs := make([]*primitive.MessageQueue, 0)
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- k := key.(primitive.MessageQueue)
- mqs = append(mqs, &k)
- return true
- })
- dc.storage.persist(mqs)
- return nil
-}
-
-func (dc *defaultConsumer) updateOffset(queue *primitive.MessageQueue, offset int64) error {
- dc.storage.update(queue, offset, false)
- return nil
-}
-
-func (dc *defaultConsumer) subscriptionAutomatically(topic string) {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if !exist {
- s := MessageSelector{
- Expression: _SubAll,
- }
- dc.subscriptionDataTable.Store(topic, buildSubscriptionData(topic, s))
- }
-}
-
-func (dc *defaultConsumer) updateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if exist {
- dc.topicSubscribeInfoTable.Store(topic, mqs)
- }
-}
-
-func (dc *defaultConsumer) isSubscribeTopicNeedUpdate(topic string) bool {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if !exist {
- return false
- }
- _, exist = dc.topicSubscribeInfoTable.Load(topic)
- return !exist
-}
-
-func (dc *defaultConsumer) doBalance() {
- dc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- topic := key.(string)
- v, exist := dc.topicSubscribeInfoTable.Load(topic)
- if !exist {
- rlog.Warning("do balance in group failed, the topic does not exist", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- })
- return true
- }
- mqs := v.([]*primitive.MessageQueue)
- switch dc.model {
- case BroadCasting:
- changed := dc.updateProcessQueueTable(topic, mqs)
- if changed {
- dc.mqChanged(topic, mqs, mqs)
- rlog.Debug("MessageQueue changed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- rlog.LogKeyMessageQueue: fmt.Sprintf("%v", mqs),
- })
- }
- case Clustering:
- cidAll := dc.findConsumerList(topic)
- if cidAll == nil {
- rlog.Warning("do balance in group failed, get consumer id list failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- })
- return true
- }
- mqAll := make([]*primitive.MessageQueue, len(mqs))
- copy(mqAll, mqs)
- sort.Strings(cidAll)
- sort.SliceStable(mqAll, func(i, j int) bool {
- v := strings.Compare(mqAll[i].Topic, mqAll[j].Topic)
- if v != 0 {
- return v < 0
- }
-
- v = strings.Compare(mqAll[i].BrokerName, mqAll[j].BrokerName)
- if v != 0 {
- return v < 0
- }
- return (mqAll[i].QueueId - mqAll[j].QueueId) < 0
- })
- allocateResult := dc.allocate(dc.consumerGroup, dc.client.ClientID(), mqAll, cidAll)
- changed := dc.updateProcessQueueTable(topic, allocateResult)
- if changed {
- dc.mqChanged(topic, mqAll, allocateResult)
- rlog.Debug("MessageQueue do balance done", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- "clientID": dc.client.ClientID(),
- "mqAllSize": len(mqAll),
- "cidAllSize": len(cidAll),
- "rebalanceResultSize": len(allocateResult),
- "rebalanceResultSet": allocateResult,
- })
- }
- }
- return true
- })
-}
-
-func (dc *defaultConsumer) SubscriptionDataList() []*internal.SubscriptionData {
- result := make([]*internal.SubscriptionData, 0)
- dc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- result = append(result, value.(*internal.SubscriptionData))
- return true
- })
- return result
-}
-
-func (dc *defaultConsumer) makeSureStateOK() error {
- if atomic.LoadInt32(&dc.state) != int32(internal.StateRunning) {
- return fmt.Errorf("state not running, actually: %v", dc.state)
- }
- return nil
-}
-
-type lockBatchRequestBody struct {
- ConsumerGroup string `json:"consumerGroup"`
- ClientId string `json:"clientId"`
- MQs []*primitive.MessageQueue `json:"mqSet"`
-}
-
-func (dc *defaultConsumer) lock(mq *primitive.MessageQueue) bool {
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, internal.MasterId, true)
-
- if brokerResult == nil {
- return false
- }
-
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: []*primitive.MessageQueue{mq},
- }
- lockedMQ := dc.doLock(brokerResult.BrokerAddr, body)
- var lockOK bool
- for idx := range lockedMQ {
- _mq := lockedMQ[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(true)
- pq.UpdateLastConsumeTime()
- pq.UpdateLastLockTime()
- }
- if _mq == *mq {
- lockOK = true
- }
- }
- fields := map[string]interface{}{
- "lockOK": lockOK,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- }
- if lockOK {
- rlog.Debug("lock MessageQueue", fields)
- } else {
- rlog.Info("lock MessageQueue", fields)
- }
- return lockOK
-}
-
-func (dc *defaultConsumer) unlock(mq *primitive.MessageQueue, oneway bool) {
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, internal.MasterId, true)
-
- if brokerResult == nil {
- return
- }
-
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: []*primitive.MessageQueue{mq},
- }
- dc.doUnlock(brokerResult.BrokerAddr, body, oneway)
- rlog.Info("unlock MessageQueue", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- "clientID": dc.client.ClientID(),
- rlog.LogKeyMessageQueue: mq.String(),
- })
-}
-
-func (dc *defaultConsumer) lockAll() {
- mqMapSet := dc.buildProcessQueueTableByBrokerName()
- for broker, mqs := range mqMapSet {
- if len(mqs) == 0 {
- continue
- }
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(broker, internal.MasterId, true)
- if brokerResult == nil {
- continue
- }
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: mqs,
- }
- lockedMQ := dc.doLock(brokerResult.BrokerAddr, body)
- set := make(map[primitive.MessageQueue]bool)
- for idx := range lockedMQ {
- _mq := lockedMQ[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(true)
- pq.UpdateLastConsumeTime()
- }
- set[_mq] = true
- }
- for idx := range mqs {
- _mq := mqs[idx]
- if !set[*_mq] {
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(false)
- pq.UpdateLastLockTime()
- rlog.Info("lock MessageQueue", map[string]interface{}{
- "lockOK": false,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: _mq.String(),
- })
- }
- }
- }
- }
-}
-
-func (dc *defaultConsumer) unlockAll(oneway bool) {
- mqMapSet := dc.buildProcessQueueTableByBrokerName()
- for broker, mqs := range mqMapSet {
- if len(mqs) == 0 {
- continue
- }
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(broker, internal.MasterId, true)
- if brokerResult == nil {
- continue
- }
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: mqs,
- }
- dc.doUnlock(brokerResult.BrokerAddr, body, oneway)
- for idx := range mqs {
- _mq := mqs[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- rlog.Info("lock MessageQueue", map[string]interface{}{
- "lockOK": false,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: _mq.String(),
- })
- v.(*processQueue).WithLock(false)
- }
- }
- }
-}
-
-func (dc *defaultConsumer) doLock(addr string, body *lockBatchRequestBody) []primitive.MessageQueue {
- data, _ := jsoniter.Marshal(body)
- request := remote.NewRemotingCommand(internal.ReqLockBatchMQ, nil, data)
- response, err := dc.client.InvokeSync(context.Background(), addr, request, 1*time.Second)
- if err != nil {
- rlog.Error("lock MessageQueue to broker invoke error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- lockOKMQSet := struct {
- MQs []primitive.MessageQueue `json:"lockOKMQSet"`
- }{}
- if len(response.Body) == 0 {
- return nil
- }
- err = jsoniter.Unmarshal(response.Body, &lockOKMQSet)
- if err != nil {
- rlog.Error("Unmarshal lock mq body error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- return lockOKMQSet.MQs
-}
-
-func (dc *defaultConsumer) doUnlock(addr string, body *lockBatchRequestBody, oneway bool) {
- data, _ := jsoniter.Marshal(body)
- request := remote.NewRemotingCommand(internal.ReqUnlockBatchMQ, nil, data)
- if oneway {
- err := dc.client.InvokeOneWay(context.Background(), addr, request, 3*time.Second)
- if err != nil {
- rlog.Error("lock MessageQueue to broker invoke oneway error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- }
- } else {
- response, err := dc.client.InvokeSync(context.Background(), addr, request, 1*time.Second)
- rlog.Error("lock MessageQueue to broker invoke error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- if response.Code != internal.ResSuccess {
- // TODO error
- }
- }
-}
-
-func (dc *defaultConsumer) buildProcessQueueTableByBrokerName() map[string][]*primitive.MessageQueue {
- result := make(map[string][]*primitive.MessageQueue, 0)
-
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- mqs, exist := result[mq.BrokerName]
- if !exist {
- mqs = make([]*primitive.MessageQueue, 0)
- }
- mqs = append(mqs, &mq)
- result[mq.BrokerName] = mqs
- return true
- })
-
- return result
-}
-
-func (dc *defaultConsumer) updateProcessQueueTable(topic string, mqs []*primitive.MessageQueue) bool {
- var changed bool
- mqSet := make(map[primitive.MessageQueue]bool)
- for idx := range mqs {
- mqSet[*mqs[idx]] = true
- }
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- if mq.Topic == topic {
- if !mqSet[mq] {
- pq.WithDropped(true)
- if dc.removeUnnecessaryMessageQueue(&mq, pq) {
- dc.processQueueTable.Delete(key)
- changed = true
- rlog.Debug("remove unnecessary mq when updateProcessQueueTable", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- } else if pq.isPullExpired() && dc.cType == _PushConsume {
- pq.WithDropped(true)
- if dc.removeUnnecessaryMessageQueue(&mq, pq) {
- dc.processQueueTable.Delete(key)
- changed = true
- rlog.Debug("remove unnecessary mq because pull was paused, prepare to fix it", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- }
- }
- return true
- })
-
- if dc.cType == _PushConsume {
- for item := range mqSet {
- // BUG: the mq will send to channel, if not copy once, the next iter will modify the mq in the channel.
- mq := item
- _, exist := dc.processQueueTable.Load(mq)
- if exist {
- continue
- }
- if dc.consumeOrderly && !dc.lock(&mq) {
- rlog.Warning("do defaultConsumer, add a new mq failed, because lock failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- continue
- }
- dc.storage.remove(&mq)
- nextOffset := dc.computePullFromWhere(&mq)
- if nextOffset >= 0 {
- _, exist := dc.processQueueTable.Load(mq)
- if exist {
- rlog.Debug("do defaultConsumer, mq already exist", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- } else {
- rlog.Debug("do defaultConsumer, add a new mq", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pq := newProcessQueue(dc.consumeOrderly)
- dc.processQueueTable.Store(mq, pq)
- pr := PullRequest{
- consumerGroup: dc.consumerGroup,
- mq: &mq,
- pq: pq,
- nextOffset: nextOffset,
- }
- dc.prCh <- pr
- changed = true
- }
- } else {
- rlog.Warning("do defaultConsumer, add a new mq failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- }
- }
-
- return changed
-}
-
-func (dc *defaultConsumer) removeUnnecessaryMessageQueue(mq *primitive.MessageQueue, pq *processQueue) bool {
- dc.storage.persist([]*primitive.MessageQueue{mq})
- dc.storage.remove(mq)
- return true
-}
-
-func (dc *defaultConsumer) computePullFromWhere(mq *primitive.MessageQueue) int64 {
- if dc.cType == _PullConsume {
- return 0
- }
- var result = int64(-1)
- lastOffset := dc.storage.read(mq, _ReadFromStore)
- if lastOffset >= 0 {
- result = lastOffset
- } else {
- switch dc.option.FromWhere {
- case ConsumeFromLastOffset:
- if lastOffset == -1 {
- if strings.HasPrefix(mq.Topic, internal.RetryGroupTopicPrefix) {
- result = 0
- } else {
- lastOffset, err := dc.queryMaxOffset(mq)
- if err == nil {
- result = lastOffset
- } else {
- rlog.Warning("query max offset error", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- rlog.LogKeyUnderlayError: err,
- })
- }
- }
- } else {
- result = -1
- }
- case ConsumeFromFirstOffset:
- if lastOffset == -1 {
- result = 0
- }
- case ConsumeFromTimestamp:
- if lastOffset == -1 {
- if strings.HasPrefix(mq.Topic, internal.RetryGroupTopicPrefix) {
- lastOffset, err := dc.queryMaxOffset(mq)
- if err == nil {
- result = lastOffset
- } else {
- result = -1
- rlog.Warning("query max offset error", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- rlog.LogKeyUnderlayError: err,
- })
- }
- } else {
- t, err := time.Parse("20060102150405", dc.option.ConsumeTimestamp)
- if err != nil {
- result = -1
- } else {
- lastOffset, err := dc.searchOffsetByTimestamp(mq, t.Unix()*1000)
- if err != nil {
- result = -1
- } else {
- result = lastOffset
- }
- }
- }
- }
- default:
- }
- }
- return result
-}
-
-func (dc *defaultConsumer) pullInner(ctx context.Context, queue *primitive.MessageQueue, data *internal.SubscriptionData,
- offset int64, numbers int, sysFlag int32, commitOffsetValue int64) (*primitive.PullResult, error) {
-
- brokerResult := dc.tryFindBroker(queue)
- if brokerResult == nil {
- rlog.Warning("no broker found for mq", map[string]interface{}{
- rlog.LogKeyMessageQueue: queue,
- })
- return nil, errors.ErrBrokerNotFound
- }
-
- if brokerResult.Slave {
- sysFlag = clearCommitOffsetFlag(sysFlag)
- }
-
- if (data.ExpType == string(TAG)) && brokerResult.BrokerVersion < internal.V4_1_0 {
- return nil, fmt.Errorf("the broker [%s, %v] does not upgrade to support for filter message by %v",
- queue.BrokerName, brokerResult.BrokerVersion, data.ExpType)
- }
-
- pullRequest := &internal.PullMessageRequestHeader{
- ConsumerGroup: dc.consumerGroup,
- Topic: queue.Topic,
- QueueId: int32(queue.QueueId),
- QueueOffset: offset,
- MaxMsgNums: int32(numbers),
- SysFlag: sysFlag,
- CommitOffset: commitOffsetValue,
- // TODO: 和java对齐
- SuspendTimeoutMillis: _BrokerSuspendMaxTime,
- SubExpression: data.SubString,
- // TODO: add subversion
- ExpressionType: string(data.ExpType),
- }
-
- if data.ExpType == string(TAG) {
- pullRequest.SubVersion = 0
- } else {
- pullRequest.SubVersion = data.SubVersion
- }
-
- // TODO: add computPullFromWhichFilterServer
-
- return dc.client.PullMessage(context.Background(), brokerResult.BrokerAddr, pullRequest)
-}
-
-func (dc *defaultConsumer) processPullResult(mq *primitive.MessageQueue, result *primitive.PullResult, data *internal.SubscriptionData) {
-
- dc.updatePullFromWhichNode(mq, result.SuggestWhichBrokerId)
-
- switch result.Status {
- case primitive.PullFound:
- result.SetMessageExts(primitive.DecodeMessage(result.GetBody()))
- msgs := result.GetMessageExts()
-
- // filter message according to tags
- msgListFilterAgain := msgs
- if data.Tags.Len() > 0 && data.ClassFilterMode {
- msgListFilterAgain = make([]*primitive.MessageExt, 0)
- for _, msg := range msgs {
- _, exist := data.Tags.Contains(msg.GetTags())
- if exist {
- msgListFilterAgain = append(msgListFilterAgain, msg)
- }
- }
- }
-
- // TODO: add filter message hook
- for _, msg := range msgListFilterAgain {
- traFlag, _ := strconv.ParseBool(msg.GetProperty(primitive.PropertyTransactionPrepared))
- if traFlag {
- msg.TransactionId = msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
- }
-
- msg.WithProperty(primitive.PropertyMinOffset, strconv.FormatInt(result.MinOffset, 10))
- msg.WithProperty(primitive.PropertyMaxOffset, strconv.FormatInt(result.MaxOffset, 10))
- }
-
- result.SetMessageExts(msgListFilterAgain)
- }
-}
-
-func (dc *defaultConsumer) findConsumerList(topic string) []string {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByTopic(topic)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByTopic(topic)
- }
-
- if brokerAddr != "" {
- req := &internal.GetConsumerListRequestHeader{
- ConsumerGroup: dc.consumerGroup,
- }
- cmd := remote.NewRemotingCommand(internal.ReqGetConsumerListByGroup, req, nil)
- res, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second) // TODO 超时机制有问题
- if err != nil {
- rlog.Error("get consumer list of group from broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyBroker: brokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- result := gjson.ParseBytes(res.Body)
- list := make([]string, 0)
- arr := result.Get("consumerIdList").Array()
- for idx := range arr {
- list = append(list, arr[idx].String())
- }
- return list
- }
- return nil
-}
-
-func (dc *defaultConsumer) sendBack(msg *primitive.MessageExt, level int) error {
- return nil
-}
-
-// QueryMaxOffset with specific queueId and topic
-func (dc *defaultConsumer) queryMaxOffset(mq *primitive.MessageQueue) (int64, error) {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- }
- if brokerAddr == "" {
- return -1, fmt.Errorf("the broker [%s] does not exist", mq.BrokerName)
- }
-
- request := &internal.GetMaxOffsetRequestHeader{
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqGetMaxOffset, request, nil)
- response, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
-
- return strconv.ParseInt(response.ExtFields["offset"], 10, 64)
-}
-
-func (dc *defaultConsumer) queryOffset(mq *primitive.MessageQueue) int64 {
- return dc.storage.read(mq, _ReadMemoryThenStore)
-}
-
-// SearchOffsetByTimestamp with specific queueId and topic
-func (dc *defaultConsumer) searchOffsetByTimestamp(mq *primitive.MessageQueue, timestamp int64) (int64, error) {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- }
- if brokerAddr == "" {
- return -1, fmt.Errorf("the broker [%s] does not exist", mq.BrokerName)
- }
-
- request := &internal.SearchOffsetRequestHeader{
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- Timestamp: timestamp,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqSearchOffsetByTimestamp, request, nil)
- response, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
-
- return strconv.ParseInt(response.ExtFields["offset"], 10, 64)
-}
-
-func buildSubscriptionData(topic string, selector MessageSelector) *internal.SubscriptionData {
- subData := &internal.SubscriptionData{
- Topic: topic,
- SubString: selector.Expression,
- ExpType: string(selector.Type),
- }
- subData.SubVersion = time.Now().UnixNano()
-
- if selector.Type != "" && selector.Type != TAG {
- return subData
- }
-
- if selector.Expression == "" || selector.Expression == _SubAll {
- subData.ExpType = string(TAG)
- subData.SubString = _SubAll
- } else {
- tags := strings.Split(selector.Expression, "||")
- subData.Tags = utils.NewSet()
- subData.Codes = utils.NewSet()
- for idx := range tags {
- trimString := strings.Trim(tags[idx], " ")
- if trimString != "" {
- if _, ok := subData.Tags.Contains(trimString); !ok {
- subData.Tags.AddKV(trimString, trimString)
- }
- hCode := utils.HashString(trimString)
- v := strconv.Itoa(hCode)
- if _, ok := subData.Codes.Contains(v); !ok {
- subData.Codes.AddKV(v, v)
- }
- }
- }
- }
- return subData
-}
-
-func buildSysFlag(commitOffset, suspend, subscription, classFilter bool) int32 {
- var flag int32 = 0
- if commitOffset {
- flag |= 0x1 << 0
- }
-
- if suspend {
- flag |= 0x1 << 1
- }
-
- if subscription {
- flag |= 0x1 << 2
- }
-
- if classFilter {
- flag |= 0x1 << 3
- }
-
- return flag
-}
-
-func clearCommitOffsetFlag(sysFlag int32) int32 {
- return sysFlag & (^0x1 << 0)
-}
-
-func (dc *defaultConsumer) tryFindBroker(mq *primitive.MessageQueue) *internal.FindBrokerResult {
- result := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, dc.recalculatePullFromWhichNode(mq), false)
- if result != nil {
- return result
- }
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- return dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, dc.recalculatePullFromWhichNode(mq), false)
-}
-
-func (dc *defaultConsumer) updatePullFromWhichNode(mq *primitive.MessageQueue, brokerId int64) {
- dc.pullFromWhichNodeTable.Store(*mq, brokerId)
-}
-
-func (dc *defaultConsumer) recalculatePullFromWhichNode(mq *primitive.MessageQueue) int64 {
- v, exist := dc.pullFromWhichNodeTable.Load(*mq)
- if exist {
- return v.(int64)
- }
- return internal.MasterId
-}
diff --git a/consumer/consumer_test.go b/consumer/consumer_test.go
deleted file mode 100644
index 12ccd18..0000000
--- a/consumer/consumer_test.go
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "sync"
- "testing"
- "time"
-
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func TestParseTimestamp(t *testing.T) {
- layout := "20060102150405"
- timestamp, err := time.ParseInLocation(layout, "20190430193409", time.UTC)
- assert.Nil(t, err)
- assert.Equal(t, int64(1556652849), timestamp.Unix())
-}
-
-func TestDoRebalance(t *testing.T) {
- Convey("Given a defaultConsumer", t, func() {
- dc := &defaultConsumer{
- model: Clustering,
- }
-
- topic := "test"
- broker := "127.0.0.1:8889"
- clientID := "clientID"
- mqs := []*primitive.MessageQueue{
- {
- Topic: topic,
- BrokerName: "",
- QueueId: 0,
- },
- {
- Topic: topic,
- BrokerName: "",
- QueueId: 1,
- },
- }
- dc.topicSubscribeInfoTable.Store(topic, mqs)
- sub := &internal.SubscriptionData{}
- dc.subscriptionDataTable.Store(topic, sub)
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- namesrvCli := internal.NewMockNamesrvs(ctrl)
- namesrvCli.EXPECT().FindBrokerAddrByTopic(gomock.Any()).Return(broker)
-
- rmqCli := internal.NewMockRMQClient(ctrl)
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- Body: []byte("{\"consumerIdList\": [\"a1\", \"a2\", \"a3\"] }"),
- }, nil)
- rmqCli.EXPECT().ClientID().Return(clientID)
- rmqCli.SetNameSrv(namesrvCli)
-
- dc.client = rmqCli
-
- var wg sync.WaitGroup
- wg.Add(1)
- dc.allocate = func(cg string, clientID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- assert.Equal(t, cidAll, []string{"a1", "a2", "a3"})
- wg.Done()
- return nil
- }
-
- dc.doBalance()
-
- wg.Wait()
- })
-}
-
-func TestComputePullFromWhere(t *testing.T) {
- Convey("Given a defaultConsumer", t, func() {
- dc := &defaultConsumer{
- model: Clustering,
- cType: _PushConsume,
- }
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- offsetStore := NewMockOffsetStore(ctrl)
- dc.storage = offsetStore
-
- mq := &primitive.MessageQueue{
- Topic: "test",
- }
-
- namesrvCli := internal.NewMockNamesrvs(ctrl)
-
- rmqCli := internal.NewMockRMQClient(ctrl)
- dc.client = rmqCli
- rmqCli.SetNameSrv(namesrvCli)
-
- Convey("get effective offset", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(10))
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(10), res)
- })
-
- Convey("ConsumeFromLastOffset for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromLastOffset
-
- broker := "a"
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return(broker)
-
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- ExtFields: map[string]string{
- "offset": "20",
- },
- }, nil)
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(20), res)
- })
-
- Convey("ConsumeFromFirstOffset for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromFirstOffset
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(0), res)
- })
-
- Convey("ConsumeFromTimestamp for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromTimestamp
-
- dc.option.ConsumeTimestamp = "20060102150405"
-
- broker := "a"
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return(broker)
-
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- ExtFields: map[string]string{
- "offset": "30",
- },
- }, nil)
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(30), res)
- })
-
- })
-}
diff --git a/consumer/interceptor.go b/consumer/interceptor.go
deleted file mode 100644
index 05ff94a..0000000
--- a/consumer/interceptor.go
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// WithTrace support rocketmq trace: https://github.com/apache/rocketmq/wiki/RIP-6-Message-Trace.
-func WithTrace(traceCfg *primitive.TraceConfig) Option {
- return func(options *consumerOptions) {
-
- ori := options.Interceptors
- options.Interceptors = make([]primitive.Interceptor, 0)
- options.Interceptors = append(options.Interceptors, newTraceInterceptor(traceCfg))
- options.Interceptors = append(options.Interceptors, ori...)
- }
-}
-
-func newTraceInterceptor(traceCfg *primitive.TraceConfig) primitive.Interceptor {
- dispatcher := internal.NewTraceDispatcher(traceCfg)
- if dispatcher != nil {
- dispatcher.Start()
- }
-
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- if dispatcher == nil {
- return fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- consumerCtx, exist := primitive.GetConsumerCtx(ctx)
- if !exist || len(consumerCtx.Msgs) == 0 {
- return next(ctx, req, reply)
- }
-
- beginT := time.Now()
- // before traceCtx
- traceCx := internal.TraceContext{
- RequestId: primitive.CreateUniqID(),
- TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
- TraceType: internal.SubBefore,
- GroupName: consumerCtx.ConsumerGroup,
- IsSuccess: true,
- }
- beans := make([]internal.TraceBean, 0)
- for _, msg := range consumerCtx.Msgs {
- if msg == nil {
- continue
- }
- regionID := msg.GetRegionID()
- traceOn := msg.IsTraceOn()
- if traceOn == "false" {
- continue
- }
- bean := internal.TraceBean{
- Topic: msg.Topic,
- MsgId: msg.MsgId,
- Tags: msg.GetTags(),
- Keys: msg.GetKeys(),
- StoreTime: msg.StoreTimestamp,
- BodyLength: int(msg.StoreSize),
- RetryTimes: int(msg.ReconsumeTimes),
- ClientHost: utils.LocalIP,
- StoreHost: utils.LocalIP,
- }
- beans = append(beans, bean)
- traceCx.RegionId = regionID
- }
- if len(beans) > 0 {
- traceCx.TraceBeans = beans
- traceCx.TimeStamp = time.Now().UnixNano() / int64(time.Millisecond)
- dispatcher.Append(traceCx)
- }
-
- err := next(ctx, req, reply)
-
- // after traceCtx
- costTime := time.Since(beginT).Nanoseconds() / int64(time.Millisecond)
- ctxType := consumerCtx.Properties[primitive.PropCtxType]
- afterCtx := internal.TraceContext{
- TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
-
- TraceType: internal.SubAfter,
- RegionId: traceCx.RegionId,
- GroupName: traceCx.GroupName,
- RequestId: traceCx.RequestId,
- IsSuccess: consumerCtx.Success,
- CostTime: costTime,
- TraceBeans: traceCx.TraceBeans,
- ContextCode: primitive.ConsumeReturnType(ctxType).Ordinal(),
- }
- dispatcher.Append(afterCtx)
- return err
- }
-}
diff --git a/consumer/lock.go b/consumer/lock.go
deleted file mode 100644
index 6fb17cd..0000000
--- a/consumer/lock.go
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "sync"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-type QueueLock struct {
- lockTable sync.Map
-}
-
-func newQueueLock() *QueueLock {
- return &QueueLock{}
-}
-
-func (ql QueueLock) fetchLock(queue primitive.MessageQueue) sync.Locker {
- v, _ := ql.lockTable.LoadOrStore(queue, new(sync.Mutex))
- return v.(*sync.Mutex)
-}
diff --git a/consumer/mock_offset_store.go b/consumer/mock_offset_store.go
deleted file mode 100644
index bac1cdb..0000000
--- a/consumer/mock_offset_store.go
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by MockGen. DO NOT EDIT.
-// Source: offset_store.go
-
-// Package consumer is a generated GoMock package.
-package consumer
-
-import (
- reflect "reflect"
-
- primitive "github.com/apache/rocketmq-client-go/v2/primitive"
- gomock "github.com/golang/mock/gomock"
-)
-
-// MockOffsetStore is a mock of OffsetStore interface
-type MockOffsetStore struct {
- ctrl *gomock.Controller
- recorder *MockOffsetStoreMockRecorder
-}
-
-// MockOffsetStoreMockRecorder is the mock recorder for MockOffsetStore
-type MockOffsetStoreMockRecorder struct {
- mock *MockOffsetStore
-}
-
-// NewMockOffsetStore creates a new mock instance
-func NewMockOffsetStore(ctrl *gomock.Controller) *MockOffsetStore {
- mock := &MockOffsetStore{ctrl: ctrl}
- mock.recorder = &MockOffsetStoreMockRecorder{mock}
- return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockOffsetStore) EXPECT() *MockOffsetStoreMockRecorder {
- return m.recorder
-}
-
-// persist mocks base method
-func (m *MockOffsetStore) persist(mqs []*primitive.MessageQueue) {
- m.ctrl.Call(m, "persist", mqs)
-}
-
-// persist indicates an expected call of persist
-func (mr *MockOffsetStoreMockRecorder) persist(mqs interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "persist", reflect.TypeOf((*MockOffsetStore)(nil).persist), mqs)
-}
-
-// remove mocks base method
-func (m *MockOffsetStore) remove(mq *primitive.MessageQueue) {
- m.ctrl.Call(m, "remove", mq)
-}
-
-// remove indicates an expected call of remove
-func (mr *MockOffsetStoreMockRecorder) remove(mq interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "remove", reflect.TypeOf((*MockOffsetStore)(nil).remove), mq)
-}
-
-// read mocks base method
-func (m *MockOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- ret := m.ctrl.Call(m, "read", mq, t)
- ret0, _ := ret[0].(int64)
- return ret0
-}
-
-// read indicates an expected call of read
-func (mr *MockOffsetStoreMockRecorder) read(mq, t interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "read", reflect.TypeOf((*MockOffsetStore)(nil).read), mq, t)
-}
-
-// update mocks base method
-func (m *MockOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- m.ctrl.Call(m, "update", mq, offset, increaseOnly)
-}
-
-// update indicates an expected call of update
-func (mr *MockOffsetStoreMockRecorder) update(mq, offset, increaseOnly interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "update", reflect.TypeOf((*MockOffsetStore)(nil).update), mq, offset, increaseOnly)
-}
diff --git a/consumer/offset_store.go b/consumer/offset_store.go
deleted file mode 100644
index 5ecfd14..0000000
--- a/consumer/offset_store.go
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "os"
- "path/filepath"
- "strconv"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- jsoniter "github.com/json-iterator/go"
-)
-
-type readType int
-
-const (
- _ReadFromMemory readType = iota
- _ReadFromStore
- _ReadMemoryThenStore
-)
-
-var (
- _LocalOffsetStorePath = os.Getenv("rocketmq.client.localOffsetStoreDir")
-)
-
-func init() {
- if _LocalOffsetStorePath == "" {
- _LocalOffsetStorePath = filepath.Join(os.Getenv("HOME"), ".rocketmq_client_go")
- }
-}
-
-//go:generate mockgen -source offset_store.go -destination mock_offset_store.go -self_package github.com/apache/rocketmq-client-go/v2/consumer --package consumer OffsetStore
-type OffsetStore interface {
- persist(mqs []*primitive.MessageQueue)
- remove(mq *primitive.MessageQueue)
- read(mq *primitive.MessageQueue, t readType) int64
- update(mq *primitive.MessageQueue, offset int64, increaseOnly bool)
-}
-
-type OffsetSerializeWrapper struct {
- OffsetTable map[MessageQueueKey]int64 `json:"offsetTable"`
-}
-
-type MessageQueueKey primitive.MessageQueue
-
-func (mq MessageQueueKey) MarshalText() (text []byte, err error) {
- repr := struct {
- Topic string `json:"topic"`
- BrokerName string `json:"brokerName"`
- QueueId int `json:"queueId"`
- }{
- Topic: mq.Topic,
- BrokerName: mq.BrokerName,
- QueueId: mq.QueueId,
- }
- text, err = jsoniter.Marshal(repr)
- return
-}
-
-func (mq *MessageQueueKey) UnmarshalText(text []byte) error {
- repr := struct {
- Topic string `json:"topic"`
- BrokerName string `json:"brokerName"`
- QueueId int `json:"queueId"`
- }{}
- err := jsoniter.Unmarshal(text, &repr)
- if err != nil {
- return err
- }
- mq.Topic = repr.Topic
- mq.QueueId = repr.QueueId
- mq.BrokerName = repr.BrokerName
-
- return nil
-}
-
-type localFileOffsetStore struct {
- group string
- path string
- OffsetTable *sync.Map // concurrent safe , map[MessageQueueKey]int64
- // mutex for offset file
- mutex sync.Mutex
-}
-
-func NewLocalFileOffsetStore(clientID, group string) OffsetStore {
- store := &localFileOffsetStore{
- group: group,
- path: filepath.Join(_LocalOffsetStorePath, clientID, group, "offset.json"),
- OffsetTable: new(sync.Map),
- }
- store.load()
- return store
-}
-
-func (local *localFileOffsetStore) load() {
- local.mutex.Lock()
- defer local.mutex.Unlock()
- data, err := utils.FileReadAll(local.path)
- if os.IsNotExist(err) {
- return
- }
- if err != nil {
- rlog.Info("read from local store error, try to use bak file", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- data, err = utils.FileReadAll(filepath.Join(local.path, ".bak"))
- }
- if err != nil {
- rlog.Info("read from local store bak file error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return
- }
- datas := make(map[MessageQueueKey]int64)
-
- wrapper := OffsetSerializeWrapper{
- OffsetTable: datas,
- }
-
- err = jsoniter.Unmarshal(data, &wrapper)
- if err != nil {
- rlog.Warning("unmarshal local offset error", map[string]interface{}{
- "local_path": local.path,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return
- }
-
- if datas != nil {
- for k, v := range datas {
- local.OffsetTable.Store(k, v)
- }
- }
-}
-
-func (local *localFileOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- switch t {
- case _ReadFromMemory, _ReadMemoryThenStore:
- off := readFromMemory(local.OffsetTable, mq)
- if off >= 0 || (off == -1 && t == _ReadFromMemory) {
- return off
- }
- fallthrough
- case _ReadFromStore:
- local.load()
- return readFromMemory(local.OffsetTable, mq)
- default:
-
- }
- return -1
-}
-
-func (local *localFileOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- local.mutex.Lock()
- defer local.mutex.Unlock()
- rlog.Debug("update offset", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- "new_offset": offset,
- })
- key := MessageQueueKey(*mq)
- localOffset, exist := local.OffsetTable.Load(key)
- if !exist {
- local.OffsetTable.Store(key, offset)
- return
- }
- if increaseOnly {
- if localOffset.(int64) < offset {
- local.OffsetTable.Store(key, offset)
- }
- } else {
- local.OffsetTable.Store(key, offset)
- }
-}
-
-func (local *localFileOffsetStore) persist(mqs []*primitive.MessageQueue) {
- if len(mqs) == 0 {
- return
- }
- local.mutex.Lock()
- defer local.mutex.Unlock()
-
- datas := make(map[MessageQueueKey]int64)
- local.OffsetTable.Range(func(key, value interface{}) bool {
- k := key.(MessageQueueKey)
- v := value.(int64)
- datas[k] = v
- return true
- })
-
- wrapper := OffsetSerializeWrapper{
- OffsetTable: datas,
- }
- data, _ := jsoniter.Marshal(wrapper)
- utils.CheckError(fmt.Sprintf("persist offset to %s", local.path), utils.WriteToFile(local.path, data))
-}
-
-func (local *localFileOffsetStore) remove(mq *primitive.MessageQueue) {
- // nothing to do
-}
-
-type remoteBrokerOffsetStore struct {
- group string
- OffsetTable map[primitive.MessageQueue]int64 `json:"OffsetTable"`
- client internal.RMQClient
- namesrv internal.Namesrvs
- mutex sync.RWMutex
-}
-
-func NewRemoteOffsetStore(group string, client internal.RMQClient, namesrv internal.Namesrvs) OffsetStore {
- return &remoteBrokerOffsetStore{
- group: group,
- client: client,
- namesrv: namesrv,
- OffsetTable: make(map[primitive.MessageQueue]int64),
- }
-}
-
-func (r *remoteBrokerOffsetStore) persist(mqs []*primitive.MessageQueue) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
- if len(mqs) == 0 {
- return
- }
-
- used := make(map[primitive.MessageQueue]struct{}, 0)
- for _, mq := range mqs {
- used[*mq] = struct{}{}
- }
-
- for mq, off := range r.OffsetTable {
- if _, ok := used[mq]; !ok {
- delete(r.OffsetTable, mq)
- continue
- }
- err := r.updateConsumeOffsetToBroker(r.group, mq, off)
- if err != nil {
- rlog.Warning("update offset to broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyUnderlayError: err.Error(),
- "offset": off,
- })
- } else {
- rlog.Info("update offset to broker success", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- "offset": off,
- })
- }
- }
-}
-
-func (r *remoteBrokerOffsetStore) remove(mq *primitive.MessageQueue) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
-
- delete(r.OffsetTable, *mq)
- rlog.Warning("delete mq from offset table", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq,
- })
-}
-
-func (r *remoteBrokerOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- r.mutex.RLock()
- switch t {
- case _ReadFromMemory, _ReadMemoryThenStore:
- off, exist := r.OffsetTable[*mq]
- if exist {
- r.mutex.RUnlock()
- return off
- }
- if t == _ReadFromMemory {
- r.mutex.RUnlock()
- return -1
- }
- fallthrough
- case _ReadFromStore:
- off, err := r.fetchConsumeOffsetFromBroker(r.group, mq)
- if err != nil {
- rlog.Error("fetch offset of mq from broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyUnderlayError: err,
- })
- r.mutex.RUnlock()
- return -1
- }
- rlog.Warning("fetch offset of mq from broker success", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- "offset": off,
- })
- r.mutex.RUnlock()
- r.update(mq, off, true)
- return off
- default:
- }
-
- return -1
-}
-
-func (r *remoteBrokerOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
- localOffset, exist := r.OffsetTable[*mq]
- if !exist {
- r.OffsetTable[*mq] = offset
- return
- }
- if increaseOnly {
- if localOffset < offset {
- r.OffsetTable[*mq] = offset
- }
- } else {
- r.OffsetTable[*mq] = offset
- }
-}
-
-func (r *remoteBrokerOffsetStore) fetchConsumeOffsetFromBroker(group string, mq *primitive.MessageQueue) (int64, error) {
- broker := r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- if broker == "" {
- r.namesrv.UpdateTopicRouteInfo(mq.Topic)
- broker = r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- }
- if broker == "" {
- return int64(-1), fmt.Errorf("broker: %s address not found", mq.BrokerName)
- }
- queryOffsetRequest := &internal.QueryConsumerOffsetRequestHeader{
- ConsumerGroup: group,
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- }
- cmd := remote.NewRemotingCommand(internal.ReqQueryConsumerOffset, queryOffsetRequest, nil)
- res, err := r.client.InvokeSync(context.Background(), broker, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
- if res.Code != internal.ResSuccess {
- return -2, fmt.Errorf("broker response code: %d, remarks: %s", res.Code, res.Remark)
- }
-
- off, err := strconv.ParseInt(res.ExtFields["offset"], 10, 64)
-
- if err != nil {
- return -1, err
- }
-
- return off, nil
-}
-
-func (r *remoteBrokerOffsetStore) updateConsumeOffsetToBroker(group string, mq primitive.MessageQueue, off int64) error {
- broker := r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- if broker == "" {
- r.namesrv.UpdateTopicRouteInfo(mq.Topic)
- broker = r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- }
- if broker == "" {
- return fmt.Errorf("broker: %s address not found", mq.BrokerName)
- }
-
- updateOffsetRequest := &internal.UpdateConsumerOffsetRequestHeader{
- ConsumerGroup: group,
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- CommitOffset: off,
- }
- cmd := remote.NewRemotingCommand(internal.ReqUpdateConsumerOffset, updateOffsetRequest, nil)
- return r.client.InvokeOneWay(context.Background(), broker, cmd, 5*time.Second)
-}
-
-func readFromMemory(table *sync.Map, mq *primitive.MessageQueue) int64 {
- localOffset, exist := table.Load(MessageQueueKey(*mq))
- if !exist {
- return -1
- }
-
- return localOffset.(int64)
-}
diff --git a/consumer/offset_store_test.go b/consumer/offset_store_test.go
deleted file mode 100644
index cfa0eaa..0000000
--- a/consumer/offset_store_test.go
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "path/filepath"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func TestNewLocalFileOffsetStore(t *testing.T) {
- Convey("Given some test cases", t, func() {
- type testCase struct {
- clientId string
- group string
- expectedResult *localFileOffsetStore
- }
- cases := []testCase{
- {
- clientId: "",
- group: "testGroup",
- expectedResult: &localFileOffsetStore{
- group: "testGroup",
- path: filepath.Join(_LocalOffsetStorePath, "/testGroup/offset.json"),
- },
- }, {
- clientId: "192.168.24.1@default",
- group: "",
- expectedResult: &localFileOffsetStore{
- group: "",
- path: filepath.Join(_LocalOffsetStorePath, "/192.168.24.1@default/offset.json"),
- },
- }, {
- clientId: "192.168.24.1@default",
- group: "testGroup",
- expectedResult: &localFileOffsetStore{
- group: "testGroup",
- path: filepath.Join(_LocalOffsetStorePath, "/192.168.24.1@default/testGroup/offset.json"),
- },
- },
- }
-
- for _, value := range cases {
- result := NewLocalFileOffsetStore(value.clientId, value.group).(*localFileOffsetStore)
- value.expectedResult.OffsetTable = result.OffsetTable
- So(result, ShouldResemble, value.expectedResult)
- }
- })
-}
-
-func TestLocalFileOffsetStore(t *testing.T) {
- Convey("Given a local store with a starting value", t, func() {
- localStore := NewLocalFileOffsetStore("192.168.24.1@default", "testGroup")
-
- type offsetCase struct {
- queue *primitive.MessageQueue
- setOffset int64
- expectedOffset int64
- }
- mq := &primitive.MessageQueue{
- Topic: "testTopic",
- BrokerName: "default",
- QueueId: 1,
- }
-
- Convey("test update", func() {
- Convey("when increaseOnly is false", func() {
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 1,
- },
- }
- for _, value := range cases {
- localStore.update(value.queue, value.setOffset, false)
- offset := localStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
-
- Convey("when increaseOnly is true", func() {
- localStore.update(mq, 0, false)
-
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 3,
- },
- }
- for _, value := range cases {
- localStore.update(value.queue, value.setOffset, true)
- offset := localStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
- })
-
- Convey("test persist", func() {
- localStore.update(mq, 1, false)
- offset := localStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, 1)
-
- queues := []*primitive.MessageQueue{mq}
- localStore.persist(queues)
- offset = localStore.read(mq, _ReadFromStore)
- So(offset, ShouldEqual, 1)
-
- localStore.(*localFileOffsetStore).OffsetTable.Delete(MessageQueueKey(*mq))
- offset = localStore.read(mq, _ReadMemoryThenStore)
- So(offset, ShouldEqual, 1)
- })
- })
-}
-
-func TestRemoteBrokerOffsetStore(t *testing.T) {
- Convey("Given a remote store with a starting value", t, func() {
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- namesrv := internal.NewMockNamesrvs(ctrl)
-
- rmqClient := internal.NewMockRMQClient(ctrl)
- remoteStore := NewRemoteOffsetStore("testGroup", rmqClient, namesrv)
-
- type offsetCase struct {
- queue *primitive.MessageQueue
- setOffset int64
- expectedOffset int64
- }
- mq := &primitive.MessageQueue{
- Topic: "testTopic",
- BrokerName: "default",
- QueueId: 1,
- }
-
- Convey("test update", func() {
- Convey("when increaseOnly is false", func() {
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 1,
- },
- }
- for _, value := range cases {
- remoteStore.update(value.queue, value.setOffset, false)
- offset := remoteStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
-
- Convey("when increaseOnly is true", func() {
- remoteStore.update(mq, 0, false)
-
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 3,
- },
- }
- for _, value := range cases {
- remoteStore.update(value.queue, value.setOffset, true)
- offset := remoteStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
- })
-
- Convey("test persist", func() {
- queues := []*primitive.MessageQueue{mq}
-
- namesrv.EXPECT().FindBrokerAddrByName(gomock.Any()).Return("192.168.24.1:10911").MaxTimes(2)
-
- ret := &remote.RemotingCommand{
- Code: internal.ResSuccess,
- ExtFields: map[string]string{
- "offset": "1",
- },
- }
- rmqClient.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(ret, nil).MaxTimes(2)
-
- remoteStore.persist(queues)
- offset := remoteStore.read(mq, _ReadFromStore)
- So(offset, ShouldEqual, 1)
-
- remoteStore.remove(mq)
- offset = remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, -1)
- offset = remoteStore.read(mq, _ReadMemoryThenStore)
- So(offset, ShouldEqual, 1)
-
- })
-
- Convey("test remove", func() {
- remoteStore.update(mq, 1, false)
- offset := remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, 1)
-
- remoteStore.remove(mq)
- offset = remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, -1)
- })
- })
-}
diff --git a/consumer/option.go b/consumer/option.go
deleted file mode 100644
index 330f6c4..0000000
--- a/consumer/option.go
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-type consumerOptions struct {
- internal.ClientOptions
-
- /**
- * Backtracking consumption time with second precision. Time format is
- * 20131223171201<br>
- * Implying Seventeen twelve and 01 seconds on December 23, 2013 year<br>
- * Default backtracking consumption time Half an hour ago.
- */
- ConsumeTimestamp string
-
- // The socket timeout in milliseconds
- ConsumerPullTimeout time.Duration
-
- // Concurrently max span offset.it has no effect on sequential consumption
- ConsumeConcurrentlyMaxSpan int
-
- // Flow control threshold on queue level, each message queue will cache at most 1000 messages by default,
- // Consider the {PullBatchSize}, the instantaneous value may exceed the limit
- PullThresholdForQueue int64
-
- // Limit the cached message size on queue level, each message queue will cache at most 100 MiB messages by default,
- // Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit
- //
- // The size of a message only measured by message body, so it's not accurate
- PullThresholdSizeForQueue int
-
- // Flow control threshold on topic level, default value is -1(Unlimited)
- //
- // The value of {@code pullThresholdForQueue} will be overwrote and calculated based on
- // {@code pullThresholdForTopic} if it is't unlimited
- //
- // For example, if the value of pullThresholdForTopic is 1000 and 10 message queues are assigned to this consumer,
- // then pullThresholdForQueue will be set to 100
- PullThresholdForTopic int
-
- // Limit the cached message size on topic level, default value is -1 MiB(Unlimited)
- //
- // The value of {@code pullThresholdSizeForQueue} will be overwrote and calculated based on
- // {@code pullThresholdSizeForTopic} if it is't unlimited
- //
- // For example, if the value of pullThresholdSizeForTopic is 1000 MiB and 10 message queues are
- // assigned to this consumer, then pullThresholdSizeForQueue will be set to 100 MiB
- PullThresholdSizeForTopic int
-
- // Message pull Interval
- PullInterval time.Duration
-
- // Batch consumption size
- ConsumeMessageBatchMaxSize int
-
- // Batch pull size
- PullBatchSize int32
-
- // Whether update subscription relationship when every pull
- PostSubscriptionWhenPull bool
-
- // Max re-consume times. -1 means 16 times.
- //
- // If messages are re-consumed more than {@link #maxReconsumeTimes} before Success, it's be directed to a deletion
- // queue waiting.
- MaxReconsumeTimes int32
-
- // Suspending pulling time for cases requiring slow pulling like flow-control scenario.
- SuspendCurrentQueueTimeMillis time.Duration
-
- // Maximum amount of time a message may block the consuming thread.
- ConsumeTimeout time.Duration
-
- ConsumerModel MessageModel
- Strategy AllocateStrategy
- ConsumeOrderly bool
- FromWhere ConsumeFromWhere
-
- Interceptors []primitive.Interceptor
- // TODO traceDispatcher
- MaxTimeConsumeContinuously time.Duration
- //
- AutoCommit bool
- RebalanceLockInterval time.Duration
-
- Resolver primitive.NsResolver
-}
-
-func defaultPushConsumerOptions() consumerOptions {
- opts := consumerOptions{
- ClientOptions: internal.DefaultClientOptions(),
- Strategy: AllocateByAveragely,
- MaxTimeConsumeContinuously: time.Duration(60 * time.Second),
- RebalanceLockInterval: 20 * time.Second,
- MaxReconsumeTimes: -1,
- ConsumerModel: Clustering,
- AutoCommit: true,
- Resolver: primitive.NewHttpResolver("DEFAULT"),
- }
- opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
- return opts
-}
-
-type Option func(*consumerOptions)
-
-func defaultPullConsumerOptions() consumerOptions {
- opts := consumerOptions{
- ClientOptions: internal.DefaultClientOptions(),
- Resolver: primitive.NewHttpResolver("DEFAULT"),
- }
- opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
- return opts
-}
-
-func WithConsumerModel(m MessageModel) Option {
- return func(options *consumerOptions) {
- options.ConsumerModel = m
- }
-}
-
-func WithConsumeFromWhere(w ConsumeFromWhere) Option {
- return func(options *consumerOptions) {
- options.FromWhere = w
- }
-}
-
-func WithConsumerOrder(order bool) Option {
- return func(options *consumerOptions) {
- options.ConsumeOrderly = order
- }
-}
-
-func WithConsumeMessageBatchMaxSize(consumeMessageBatchMaxSize int) Option {
- return func(options *consumerOptions) {
- options.ConsumeMessageBatchMaxSize = consumeMessageBatchMaxSize
- }
-}
-
-// WithChainConsumerInterceptor returns a ConsumerOption that specifies the chained interceptor for consumer.
-// The first interceptor will be the outer most, while the last interceptor will be the inner most wrapper
-// around the real call.
-func WithInterceptor(fs ...primitive.Interceptor) Option {
- return func(options *consumerOptions) {
- options.Interceptors = append(options.Interceptors, fs...)
- }
-}
-
-// WithGroupName set group name address
-func WithGroupName(group string) Option {
- return func(opts *consumerOptions) {
- if group == "" {
- return
- }
- opts.GroupName = group
- }
-}
-
-func WithInstance(name string) Option {
- return func(options *consumerOptions) {
- options.InstanceName = name
- }
-}
-
-// WithNamespace set the namespace of consumer
-func WithNamespace(namespace string) Option {
- return func(opts *consumerOptions) {
- opts.Namespace = namespace
- }
-}
-
-func WithVIPChannel(enable bool) Option {
- return func(opts *consumerOptions) {
- opts.VIPChannelEnabled = enable
- }
-}
-
-// WithRetry return a Option that specifies the retry times when send failed.
-// TODO: use retry middleware instead
-func WithRetry(retries int) Option {
- return func(opts *consumerOptions) {
- opts.RetryTimes = retries
- }
-}
-
-func WithCredentials(c primitive.Credentials) Option {
- return func(options *consumerOptions) {
- options.ClientOptions.Credentials = c
- }
-}
-
-// WithMaxReconsumeTimes set MaxReconsumeTimes of options, if message reconsume greater than MaxReconsumeTimes, it will
-// be sent to retry or dlq topic. more info reference by examples/consumer/retry.
-func WithMaxReconsumeTimes(times int32) Option {
- return func(opts *consumerOptions) {
- opts.MaxReconsumeTimes = times
- }
-}
-
-func WithStrategy(strategy AllocateStrategy) Option {
- return func(opts *consumerOptions) {
- opts.Strategy = strategy
- }
-}
-
-func WithPullBatchSize(batchSize int32) Option {
- return func(options *consumerOptions) {
- options.PullBatchSize = batchSize
- }
-}
-
-func WithRebalanceLockInterval(interval time.Duration) Option {
- return func(options *consumerOptions) {
- options.RebalanceLockInterval = interval
- }
-}
-
-func WithAutoCommit(auto bool) Option {
- return func(options *consumerOptions) {
- options.AutoCommit = auto
- }
-}
-
-func WithSuspendCurrentQueueTimeMillis(suspendT time.Duration) Option {
- return func(options *consumerOptions) {
- options.SuspendCurrentQueueTimeMillis = suspendT
- }
-}
-
-func WithPullInterval(interval time.Duration) Option {
- return func(options *consumerOptions) {
- options.PullInterval = interval
- }
-}
-
-// WithNsResolver set nameserver resolver to fetch nameserver addr
-func WithNsResolver(resolver primitive.NsResolver) Option {
- return func(options *consumerOptions) {
- options.Resolver = resolver
- }
-}
-
-// WithNameServer set NameServer address, only support one NameServer cluster in alpha2
-func WithNameServer(nameServers primitive.NamesrvAddr) Option {
- return func(options *consumerOptions) {
- options.Resolver = primitive.NewPassthroughResolver(nameServers)
- }
-}
-
-// WithNameServerDomain set NameServer domain
-func WithNameServerDomain(nameServerUrl string) Option {
- return func(opts *consumerOptions) {
- opts.Resolver = primitive.NewHttpResolver("DEFAULT", nameServerUrl)
- }
-}
diff --git a/consumer/process_queue.go b/consumer/process_queue.go
deleted file mode 100644
index 76a9236..0000000
--- a/consumer/process_queue.go
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "strconv"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/emirpasic/gods/maps/treemap"
- "github.com/emirpasic/gods/utils"
- gods_util "github.com/emirpasic/gods/utils"
- uatomic "go.uber.org/atomic"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- _RebalanceLockMaxTime = 30 * time.Second
- _RebalanceInterval = 20 * time.Second
- _PullMaxIdleTime = 120 * time.Second
-)
-
-type processQueue struct {
- cachedMsgCount int64
- cachedMsgSize int64
- tryUnlockTimes int64
- queueOffsetMax int64
- msgAccCnt int64
- msgCache *treemap.Map
- mutex sync.RWMutex
- consumeLock sync.Mutex
- consumingMsgOrderlyTreeMap *treemap.Map
- dropped *uatomic.Bool
- lastPullTime atomic.Value
- lastConsumeTime atomic.Value
- locked *uatomic.Bool
- lastLockTime atomic.Value
- consuming bool
- lockConsume sync.Mutex
- msgCh chan []*primitive.MessageExt
- order bool
-}
-
-func newProcessQueue(order bool) *processQueue {
- consumingMsgOrderlyTreeMap := treemap.NewWith(gods_util.Int64Comparator)
-
- lastConsumeTime := atomic.Value{}
- lastConsumeTime.Store(time.Now())
-
- lastLockTime := atomic.Value{}
- lastLockTime.Store(time.Now())
-
- lastPullTime := atomic.Value{}
- lastPullTime.Store(time.Now())
-
- pq := &processQueue{
- msgCache: treemap.NewWith(utils.Int64Comparator),
- lastPullTime: lastPullTime,
- lastConsumeTime: lastConsumeTime,
- lastLockTime: lastLockTime,
- msgCh: make(chan []*primitive.MessageExt, 32),
- consumingMsgOrderlyTreeMap: consumingMsgOrderlyTreeMap,
- order: order,
- locked: uatomic.NewBool(false),
- dropped: uatomic.NewBool(false),
- }
- return pq
-}
-
-func (pq *processQueue) putMessage(messages ...*primitive.MessageExt) {
- if len(messages) == 0 {
- return
- }
- pq.mutex.Lock()
- if pq.IsDroppd() {
- pq.mutex.Unlock()
- return
- }
- if !pq.order {
- pq.msgCh <- messages
- }
- validMessageCount := 0
- for idx := range messages {
- msg := messages[idx]
- _, found := pq.msgCache.Get(msg.QueueOffset)
- if found {
- continue
- }
- _, found = pq.consumingMsgOrderlyTreeMap.Get(msg.QueueOffset)
- if found {
- continue
- }
- pq.msgCache.Put(msg.QueueOffset, msg)
- validMessageCount++
- pq.queueOffsetMax = msg.QueueOffset
- atomic.AddInt64(&pq.cachedMsgSize, int64(len(msg.Body)))
- }
- pq.mutex.Unlock()
-
- atomic.AddInt64(&pq.cachedMsgCount, int64(validMessageCount))
-
- if pq.msgCache.Size() > 0 && !pq.consuming {
- pq.consuming = true
- }
-
- msg := messages[len(messages)-1]
- maxOffset, err := strconv.ParseInt(msg.GetProperty(primitive.PropertyMaxOffset), 10, 64)
- if err != nil {
- acc := maxOffset - msg.QueueOffset
- if acc > 0 {
- pq.msgAccCnt = acc
- }
- }
-}
-
-func (pq *processQueue) WithLock(lock bool) {
- pq.locked.Store(lock)
-}
-
-func (pq *processQueue) IsLock() bool {
- return pq.locked.Load()
-}
-
-func (pq *processQueue) WithDropped(dropped bool) {
- pq.dropped.Store(dropped)
-}
-
-func (pq *processQueue) IsDroppd() bool {
- return pq.dropped.Load()
-}
-
-func (pq *processQueue) UpdateLastConsumeTime() {
- pq.lastConsumeTime.Store(time.Now())
-}
-
-func (pq *processQueue) LastConsumeTime() time.Time {
- return pq.lastConsumeTime.Load().(time.Time)
-}
-
-func (pq *processQueue) UpdateLastLockTime() {
- pq.lastLockTime.Store(time.Now())
-}
-
-func (pq *processQueue) LastLockTime() time.Time {
- return pq.lastLockTime.Load().(time.Time)
-}
-
-func (pq *processQueue) LastPullTime() time.Time {
- return pq.lastPullTime.Load().(time.Time)
-}
-
-func (pq *processQueue) UpdateLastPullTime() {
- pq.lastPullTime.Store(time.Now())
-}
-
-func (pq *processQueue) makeMessageToCosumeAgain(messages ...*primitive.MessageExt) {
- pq.mutex.Lock()
- for _, msg := range messages {
- pq.consumingMsgOrderlyTreeMap.Remove(msg.QueueOffset)
- pq.msgCache.Put(msg.QueueOffset, msg)
- }
-
- pq.mutex.Unlock()
-}
-
-func (pq *processQueue) removeMessage(messages ...*primitive.MessageExt) int64 {
- result := int64(-1)
- pq.mutex.Lock()
- pq.UpdateLastConsumeTime()
- if !pq.msgCache.Empty() {
- result = pq.queueOffsetMax + 1
- removedCount := 0
- for idx := range messages {
- msg := messages[idx]
- _, found := pq.msgCache.Get(msg.QueueOffset)
- if !found {
- continue
- }
- pq.msgCache.Remove(msg.QueueOffset)
- removedCount++
- atomic.AddInt64(&pq.cachedMsgSize, int64(-len(msg.Body)))
- }
- atomic.AddInt64(&pq.cachedMsgCount, int64(-removedCount))
- }
- if !pq.msgCache.Empty() {
- first, _ := pq.msgCache.Min()
- result = first.(int64)
- }
- pq.mutex.Unlock()
- return result
-}
-
-func (pq *processQueue) isLockExpired() bool {
- return time.Now().Sub(pq.LastLockTime()) > _RebalanceLockMaxTime
-}
-
-func (pq *processQueue) isPullExpired() bool {
- return time.Now().Sub(pq.LastPullTime()) > _PullMaxIdleTime
-}
-
-func (pq *processQueue) cleanExpiredMsg(consumer defaultConsumer) {
- if consumer.option.ConsumeOrderly {
- return
- }
- var loop = 16
- if pq.msgCache.Size() < 16 {
- loop = pq.msgCache.Size()
- }
-
- for i := 0; i < loop; i++ {
- pq.mutex.RLock()
- if pq.msgCache.Empty() {
- pq.mutex.RLock()
- return
- }
- _, firstValue := pq.msgCache.Min()
- msg := firstValue.(*primitive.MessageExt)
- startTime := msg.GetProperty(primitive.PropertyConsumeStartTime)
- if startTime != "" {
- st, err := strconv.ParseInt(startTime, 10, 64)
- if err != nil {
- rlog.Warning("parse message start consume time error", map[string]interface{}{
- "time": startTime,
- rlog.LogKeyUnderlayError: err,
- })
- continue
- }
- if time.Now().Unix()-st <= int64(consumer.option.ConsumeTimeout) {
- pq.mutex.RLock()
- return
- }
- }
- pq.mutex.RLock()
-
- err := consumer.sendBack(msg, 3)
- if err != nil {
- rlog.Error("send message back to broker error when clean expired messages", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- continue
- }
- pq.removeMessage(msg)
- }
-}
-
-func (pq *processQueue) getMaxSpan() int {
- pq.mutex.RLock()
- defer pq.mutex.RUnlock()
- if pq.msgCache.Size() == 0 {
- return 0
- }
- firstKey, _ := pq.msgCache.Min()
- lastKey, _ := pq.msgCache.Max()
- return int(lastKey.(int64) - firstKey.(int64))
-}
-
-func (pq *processQueue) getMessages() []*primitive.MessageExt {
- return <-pq.msgCh
-}
-
-func (pq *processQueue) takeMessages(number int) []*primitive.MessageExt {
- for pq.msgCache.Empty() {
- time.Sleep(10 * time.Millisecond)
- }
- result := make([]*primitive.MessageExt, number)
- i := 0
- pq.mutex.Lock()
- for ; i < number; i++ {
- k, v := pq.msgCache.Min()
- if v == nil {
- break
- }
- result[i] = v.(*primitive.MessageExt)
- pq.consumingMsgOrderlyTreeMap.Put(k, v)
- pq.msgCache.Remove(k)
- }
- pq.mutex.Unlock()
- return result[:i]
-}
-
-func (pq *processQueue) Min() int64 {
- if pq.msgCache.Empty() {
- return -1
- }
- k, _ := pq.msgCache.Min()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) Max() int64 {
- if pq.msgCache.Empty() {
- return -1
- }
- k, _ := pq.msgCache.Max()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) MinOrderlyCache() int64 {
- if pq.consumingMsgOrderlyTreeMap.Empty() {
- return -1
- }
- k, _ := pq.consumingMsgOrderlyTreeMap.Min()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) MaxOrderlyCache() int64 {
- if pq.consumingMsgOrderlyTreeMap.Empty() {
- return -1
- }
- k, _ := pq.consumingMsgOrderlyTreeMap.Max()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) clear() {
- pq.mutex.Lock()
- defer pq.mutex.Unlock()
- pq.msgCache.Clear()
- pq.cachedMsgCount = 0
- pq.cachedMsgSize = 0
- pq.queueOffsetMax = 0
-}
-
-func (pq *processQueue) commit() int64 {
- pq.mutex.Lock()
- defer pq.mutex.Unlock()
-
- var offset int64
- iter, _ := pq.consumingMsgOrderlyTreeMap.Max()
- if iter != nil {
- offset = iter.(int64)
- }
- pq.cachedMsgCount -= int64(pq.consumingMsgOrderlyTreeMap.Size())
- pq.consumingMsgOrderlyTreeMap.Each(func(key interface{}, value interface{}) {
- msg := value.(*primitive.MessageExt)
- pq.cachedMsgSize -= int64(len(msg.Body))
- })
- pq.consumingMsgOrderlyTreeMap.Clear()
- return offset + 1
-}
-
-func (pq *processQueue) currentInfo() internal.ProcessQueueInfo {
- pq.mutex.RLock()
- defer pq.mutex.RUnlock()
- info := internal.ProcessQueueInfo{
- Locked: pq.locked.Load(),
- TryUnlockTimes: pq.tryUnlockTimes,
- LastLockTimestamp: pq.LastLockTime().UnixNano() / int64(time.Millisecond),
- Dropped: pq.dropped.Load(),
- LastPullTimestamp: pq.LastPullTime().UnixNano() / int64(time.Millisecond),
- LastConsumeTimestamp: pq.LastConsumeTime().UnixNano() / int64(time.Millisecond),
- }
-
- if !pq.msgCache.Empty() {
- info.CachedMsgMinOffset = pq.Min()
- info.CachedMsgMaxOffset = pq.Max()
- info.CachedMsgCount = pq.msgCache.Size()
- info.CachedMsgSizeInMiB = pq.cachedMsgSize / int64(1024*1024)
- }
-
- if !pq.consumingMsgOrderlyTreeMap.Empty() {
- info.TransactionMsgMinOffset = pq.MinOrderlyCache()
- info.TransactionMsgMaxOffset = pq.MaxOrderlyCache()
- info.TransactionMsgCount = pq.consumingMsgOrderlyTreeMap.Size()
- }
-
- return info
-}
diff --git a/consumer/pull_consumer.go b/consumer/pull_consumer.go
deleted file mode 100644
index 874973b..0000000
--- a/consumer/pull_consumer.go
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- errors2 "github.com/apache/rocketmq-client-go/v2/errors"
- "sync"
- "sync/atomic"
-
- "github.com/pkg/errors"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type PullConsumer interface {
- // Start
- Start()
-
- // Shutdown refuse all new pull operation, finish all submitted.
- Shutdown()
-
- // Pull pull message of topic, selector indicate which queue to pull.
- Pull(ctx context.Context, topic string, selector MessageSelector, numbers int) (*primitive.PullResult, error)
-
- // PullFrom pull messages of queue from the offset to offset + numbers
- PullFrom(ctx context.Context, queue *primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error)
-
- // updateOffset update offset of queue in mem
- UpdateOffset(queue *primitive.MessageQueue, offset int64) error
-
- // PersistOffset persist all offset in mem.
- PersistOffset(ctx context.Context) error
-
- // CurrentOffset return the current offset of queue in mem.
- CurrentOffset(queue *primitive.MessageQueue) (int64, error)
-}
-
-var (
- queueCounterTable sync.Map
-)
-
-type defaultPullConsumer struct {
- *defaultConsumer
-
- option consumerOptions
- client internal.RMQClient
- GroupName string
- Model MessageModel
- UnitMode bool
-
- interceptor primitive.Interceptor
-}
-
-func NewPullConsumer(options ...Option) (*defaultPullConsumer, error) {
- defaultOpts := defaultPullConsumerOptions()
- for _, apply := range options {
- apply(&defaultOpts)
- }
-
- srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
- if err != nil {
- return nil, errors.Wrap(err, "new Namesrv failed.")
- }
-
- defaultOpts.Namesrv = srvs
- dc := &defaultConsumer{
- client: internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil),
- consumerGroup: defaultOpts.GroupName,
- cType: _PullConsume,
- state: int32(internal.StateCreateJust),
- prCh: make(chan PullRequest, 4),
- model: defaultOpts.ConsumerModel,
- option: defaultOpts,
- }
- if dc.client == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = dc.client.GetNameSrv()
-
- c := &defaultPullConsumer{
- defaultConsumer: dc,
- }
- return c, nil
-}
-
-func (c *defaultPullConsumer) Start() error {
- atomic.StoreInt32(&c.state, int32(internal.StateRunning))
-
- var err error
- c.once.Do(func() {
- err = c.start()
- if err != nil {
- return
- }
- })
-
- return err
-}
-
-func (c *defaultPullConsumer) Pull(ctx context.Context, topic string, selector MessageSelector, numbers int) (*primitive.PullResult, error) {
- mq := c.getNextQueueOf(topic)
- if mq == nil {
- return nil, fmt.Errorf("prepard to pull topic: %s, but no queue is founded", topic)
- }
-
- data := buildSubscriptionData(mq.Topic, selector)
- result, err := c.pull(context.Background(), mq, data, c.nextOffsetOf(mq), numbers)
-
- if err != nil {
- return nil, err
- }
-
- c.processPullResult(mq, result, data)
- return result, nil
-}
-
-func (c *defaultPullConsumer) getNextQueueOf(topic string) *primitive.MessageQueue {
- queues, err := c.defaultConsumer.client.GetNameSrv().FetchSubscribeMessageQueues(topic)
- if err != nil && len(queues) > 0 {
- rlog.Error("get next mq error", map[string]interface{}{
- rlog.LogKeyTopic: topic,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return nil
- }
- var index int64
- v, exist := queueCounterTable.Load(topic)
- if !exist {
- index = -1
- queueCounterTable.Store(topic, int64(0))
- } else {
- index = v.(int64)
- }
-
- return queues[int(atomic.AddInt64(&index, 1))%len(queues)]
-}
-
-// SubscribeWithChan ack manually
-func (c *defaultPullConsumer) SubscribeWithChan(topic, selector MessageSelector) (chan *primitive.Message, error) {
- return nil, nil
-}
-
-// SubscribeWithFunc ack automatic
-func (c *defaultPullConsumer) SubscribeWithFunc(topic, selector MessageSelector,
- f func(msg *primitive.Message) ConsumeResult) error {
- return nil
-}
-
-func (c *defaultPullConsumer) ACK(msg *primitive.Message, result ConsumeResult) {
-
-}
-
-func (dc *defaultConsumer) checkPull(ctx context.Context, mq *primitive.MessageQueue, offset int64, numbers int) error {
- err := dc.makeSureStateOK()
- if err != nil {
- return err
- }
-
- if mq == nil {
- return errors2.ErrMQEmpty
- }
-
- if offset < 0 {
- return errors2.ErrOffset
- }
-
- if numbers <= 0 {
- return errors2.ErrNumbers
- }
- return nil
-}
-
-// TODO: add timeout limit
-// TODO: add hook
-func (c *defaultPullConsumer) pull(ctx context.Context, mq *primitive.MessageQueue, data *internal.SubscriptionData,
- offset int64, numbers int) (*primitive.PullResult, error) {
-
- if err := c.checkPull(ctx, mq, offset, numbers); err != nil {
- return nil, err
- }
-
- c.subscriptionAutomatically(mq.Topic)
-
- sysFlag := buildSysFlag(false, true, true, false)
-
- pullResp, err := c.pullInner(ctx, mq, data, offset, numbers, sysFlag, 0)
- if err != nil {
- return nil, err
- }
- c.processPullResult(mq, pullResp, data)
-
- return pullResp, err
-}
-
-func (c *defaultPullConsumer) makeSureStateOK() error {
- if atomic.LoadInt32(&c.state) != int32(internal.StateRunning) {
- return fmt.Errorf("the consumer state is [%d], not running", c.state)
- }
- return nil
-}
-
-func (c *defaultPullConsumer) nextOffsetOf(queue *primitive.MessageQueue) int64 {
- return c.computePullFromWhere(queue)
-}
-
-// PullFrom pull messages of queue from the offset to offset + numbers
-func (c *defaultPullConsumer) PullFrom(ctx context.Context, queue *primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error) {
- if err := c.checkPull(ctx, queue, offset, numbers); err != nil {
- return nil, err
- }
-
- selector := MessageSelector{}
- data := buildSubscriptionData(queue.Topic, selector)
-
- return c.pull(ctx, queue, data, offset, numbers)
-}
-
-// updateOffset update offset of queue in mem
-func (c *defaultPullConsumer) UpdateOffset(queue *primitive.MessageQueue, offset int64) error {
- return c.updateOffset(queue, offset)
-}
-
-// PersistOffset persist all offset in mem.
-func (c *defaultPullConsumer) PersistOffset(ctx context.Context) error {
- return c.persistConsumerOffset()
-}
-
-// CurrentOffset return the current offset of queue in mem.
-func (c *defaultPullConsumer) CurrentOffset(queue *primitive.MessageQueue) (int64, error) {
- v := c.queryOffset(queue)
- return v, nil
-}
-
-// Shutdown close defaultConsumer, refuse new request.
-func (c *defaultPullConsumer) Shutdown() error {
- return c.defaultConsumer.shutdown()
-}
diff --git a/consumer/pull_consumer_test.go b/consumer/pull_consumer_test.go
deleted file mode 100644
index f7bc454..0000000
--- a/consumer/pull_consumer_test.go
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
diff --git a/consumer/push_consumer.go b/consumer/push_consumer.go
deleted file mode 100644
index 8642aa4..0000000
--- a/consumer/push_consumer.go
+++ /dev/null
@@ -1,1254 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- errors2 "github.com/apache/rocketmq-client-go/v2/errors"
- "math"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/pkg/errors"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-// In most scenarios, this is the mostly recommended usage to consume messages.
-//
-// Technically speaking, this push client is virtually a wrapper of the underlying pull service. Specifically, on
-// arrival of messages pulled from brokers, it roughly invokes the registered callback handler to feed the messages.
-//
-// See quick start/Consumer in the example module for a typical usage.
-//
-// <strong>Thread Safety:</strong> After initialization, the instance can be regarded as thread-safe.
-
-const (
- Mb = 1024 * 1024
-)
-
-type PushConsumerCallback struct {
- topic string
- f func(context.Context, ...*primitive.MessageExt) (ConsumeResult, error)
-}
-
-func (callback PushConsumerCallback) UniqueID() string {
- return callback.topic
-}
-
-type pushConsumer struct {
- *defaultConsumer
- queueFlowControlTimes int
- queueMaxSpanFlowControlTimes int
- consumeFunc utils.Set
- submitToConsume func(*processQueue, *primitive.MessageQueue)
- subscribedTopic map[string]string
- interceptor primitive.Interceptor
- queueLock *QueueLock
- done chan struct{}
- closeOnce sync.Once
-}
-
-func NewPushConsumer(opts ...Option) (*pushConsumer, error) {
- defaultOpts := defaultPushConsumerOptions()
- for _, apply := range opts {
- apply(&defaultOpts)
- }
- srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
- if err != nil {
- return nil, errors.Wrap(err, "new Namesrv failed.")
- }
- if !defaultOpts.Credentials.IsEmpty() {
- srvs.SetCredentials(defaultOpts.Credentials)
- }
- defaultOpts.Namesrv = srvs
-
- if defaultOpts.Namespace != "" {
- defaultOpts.GroupName = defaultOpts.Namespace + "%" + defaultOpts.GroupName
- }
-
- dc := &defaultConsumer{
- client: internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil),
- consumerGroup: defaultOpts.GroupName,
- cType: _PushConsume,
- state: int32(internal.StateCreateJust),
- prCh: make(chan PullRequest, 4),
- model: defaultOpts.ConsumerModel,
- consumeOrderly: defaultOpts.ConsumeOrderly,
- fromWhere: defaultOpts.FromWhere,
- allocate: defaultOpts.Strategy,
- option: defaultOpts,
- }
- if dc.client == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = dc.client.GetNameSrv()
-
- p := &pushConsumer{
- defaultConsumer: dc,
- subscribedTopic: make(map[string]string, 0),
- queueLock: newQueueLock(),
- done: make(chan struct{}, 1),
- consumeFunc: utils.NewSet(),
- }
- dc.mqChanged = p.messageQueueChanged
- if p.consumeOrderly {
- p.submitToConsume = p.consumeMessageOrderly
- } else {
- p.submitToConsume = p.consumeMessageCurrently
- }
-
- p.interceptor = primitive.ChainInterceptors(p.option.Interceptors...)
-
- return p, nil
-}
-
-func (pc *pushConsumer) Start() error {
- var err error
- pc.once.Do(func() {
- rlog.Info("the consumer start beginning", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- "messageModel": pc.model,
- "unitMode": pc.unitMode,
- })
- atomic.StoreInt32(&pc.state, int32(internal.StateStartFailed))
- pc.validate()
-
- err = pc.client.RegisterConsumer(pc.consumerGroup, pc)
- if err != nil {
- rlog.Error("the consumer group has been created, specify another one", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- err = errors2.ErrCreated
- return
- }
-
- err = pc.defaultConsumer.start()
- if err != nil {
- return
- }
-
- go func() {
- // todo start clean msg expired
- for {
- select {
- case pr := <-pc.prCh:
- go func() {
- pc.pullMessage(&pr)
- }()
- case <-pc.done:
- rlog.Info("push consumer close pullConsumer listener.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- }()
-
- go primitive.WithRecover(func() {
- // initial lock.
- if !pc.consumeOrderly {
- return
- }
-
- time.Sleep(1000 * time.Millisecond)
- pc.lockAll()
-
- lockTicker := time.NewTicker(pc.option.RebalanceLockInterval)
- defer lockTicker.Stop()
- for {
- select {
- case <-lockTicker.C:
- pc.lockAll()
- case <-pc.done:
- rlog.Info("push consumer close tick.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- })
- })
-
- if err != nil {
- return err
- }
-
- pc.client.UpdateTopicRouteInfo()
- for k := range pc.subscribedTopic {
- _, exist := pc.topicSubscribeInfoTable.Load(k)
- if !exist {
- pc.client.Shutdown()
- return fmt.Errorf("the topic=%s route info not found, it may not exist", k)
- }
- }
- pc.client.CheckClientInBroker()
- pc.client.SendHeartbeatToAllBrokerWithLock()
- pc.client.RebalanceImmediately()
-
- return err
-}
-
-func (pc *pushConsumer) Shutdown() error {
- var err error
- pc.closeOnce.Do(func() {
- close(pc.done)
-
- pc.client.UnregisterConsumer(pc.consumerGroup)
- err = pc.defaultConsumer.shutdown()
- })
-
- return err
-}
-
-func (pc *pushConsumer) Subscribe(topic string, selector MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (ConsumeResult, error)) error {
- if atomic.LoadInt32(&pc.state) == int32(internal.StateStartFailed) ||
- atomic.LoadInt32(&pc.state) == int32(internal.StateShutdown) {
- return errors2.ErrStartTopic
- }
-
- if pc.option.Namespace != "" {
- topic = pc.option.Namespace + "%" + topic
- }
- data := buildSubscriptionData(topic, selector)
- pc.subscriptionDataTable.Store(topic, data)
- pc.subscribedTopic[topic] = ""
-
- pc.consumeFunc.Add(&PushConsumerCallback{
- f: f,
- topic: topic,
- })
- return nil
-}
-
-func (pc *pushConsumer) Unsubscribe(topic string) error {
- if pc.option.Namespace != "" {
- topic = pc.option.Namespace + "%" + topic
- }
- pc.subscriptionDataTable.Delete(topic)
- return nil
-}
-
-func (pc *pushConsumer) Rebalance() {
- pc.defaultConsumer.doBalance()
-}
-
-func (pc *pushConsumer) PersistConsumerOffset() error {
- return pc.defaultConsumer.persistConsumerOffset()
-}
-
-func (pc *pushConsumer) UpdateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
- pc.defaultConsumer.updateTopicSubscribeInfo(topic, mqs)
-}
-
-func (pc *pushConsumer) IsSubscribeTopicNeedUpdate(topic string) bool {
- return pc.defaultConsumer.isSubscribeTopicNeedUpdate(topic)
-}
-
-func (pc *pushConsumer) SubscriptionDataList() []*internal.SubscriptionData {
- return pc.defaultConsumer.SubscriptionDataList()
-}
-
-func (pc *pushConsumer) IsUnitMode() bool {
- return pc.unitMode
-}
-
-func (pc *pushConsumer) GetcType() string {
- return string(pc.cType)
-}
-
-func (pc *pushConsumer) GetModel() string {
- return pc.model.String()
-}
-
-func (pc *pushConsumer) GetWhere() string {
- switch pc.fromWhere {
- case ConsumeFromLastOffset:
- return "CONSUME_FROM_LAST_OFFSET"
- case ConsumeFromFirstOffset:
- return "CONSUME_FROM_FIRST_OFFSET"
- case ConsumeFromTimestamp:
- return "CONSUME_FROM_TIMESTAMP"
- default:
- return "UNKOWN"
- }
-
-}
-
-func (pc *pushConsumer) ConsumeMessageDirectly(msg *primitive.MessageExt, brokerName string) *internal.ConsumeMessageDirectlyResult {
- var msgs = []*primitive.MessageExt{msg}
- var mq = &primitive.MessageQueue{
- Topic: msg.Topic,
- BrokerName: brokerName,
- QueueId: msg.Queue.QueueId,
- }
-
- beginTime := time.Now()
- pc.resetRetryAndNamespace(msgs)
- var result ConsumeResult
-
- var err error
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: msgs,
- }
- ctx := context.Background()
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
- concurrentCtx := primitive.NewConsumeConcurrentlyContext()
- concurrentCtx.MQ = *mq
- ctx = primitive.WithConcurrentlyCtx(ctx, concurrentCtx)
-
- result, err = pc.consumeInner(ctx, msgs)
-
- consumeRT := time.Now().Sub(beginTime)
-
- res := &internal.ConsumeMessageDirectlyResult{
- Order: false,
- AutoCommit: true,
- SpentTimeMills: int64(consumeRT / time.Millisecond),
- }
-
- if err != nil {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.ExceptionReturn)
- res.ConsumeResult = internal.ThrowException
- res.Remark = err.Error()
- } else if result == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- res.ConsumeResult = internal.ConsumeSuccess
- } else if result == ConsumeRetryLater {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- res.ConsumeResult = internal.ConsumeRetryLater
- }
-
- pc.stat.increaseConsumeRT(pc.consumerGroup, mq.Topic, int64(consumeRT/time.Millisecond))
-
- return res
-}
-
-func (pc *pushConsumer) GetConsumerRunningInfo() *internal.ConsumerRunningInfo {
- info := internal.NewConsumerRunningInfo()
-
- pc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- topic := key.(string)
- info.SubscriptionData[value.(*internal.SubscriptionData)] = true
- status := internal.ConsumeStatus{
- PullRT: pc.stat.getPullRT(topic, pc.consumerGroup).avgpt,
- PullTPS: pc.stat.getPullTPS(topic, pc.consumerGroup).tps,
- ConsumeRT: pc.stat.getConsumeRT(topic, pc.consumerGroup).avgpt,
- ConsumeOKTPS: pc.stat.getConsumeOKTPS(topic, pc.consumerGroup).tps,
- ConsumeFailedTPS: pc.stat.getConsumeFailedTPS(topic, pc.consumerGroup).tps,
- ConsumeFailedMsgs: pc.stat.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + pc.consumerGroup).sum,
- }
- info.StatusTable[topic] = status
- return true
- })
-
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- pInfo := pq.currentInfo()
- pInfo.CommitOffset = pc.storage.read(&mq, _ReadMemoryThenStore)
- info.MQTable[mq] = pInfo
- return true
- })
-
- nsAddr := ""
- for _, value := range pc.client.GetNameSrv().AddrList() {
- nsAddr += fmt.Sprintf("%s;", value)
- }
- info.Properties[internal.PropNameServerAddr] = nsAddr
- info.Properties[internal.PropConsumeType] = string(pc.cType)
- info.Properties[internal.PropConsumeOrderly] = strconv.FormatBool(pc.consumeOrderly)
- info.Properties[internal.PropThreadPoolCoreSize] = "-1"
- info.Properties[internal.PropConsumerStartTimestamp] = strconv.FormatInt(pc.consumerStartTimestamp, 10)
- return info
-}
-
-func (pc *pushConsumer) messageQueueChanged(topic string, mqAll, mqDivided []*primitive.MessageQueue) {
- v, exit := pc.subscriptionDataTable.Load(topic)
- if !exit {
- return
- }
- data := v.(*internal.SubscriptionData)
- newVersion := time.Now().UnixNano()
- rlog.Info("the MessageQueue changed, version also updated", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: data.SubVersion,
- rlog.LogKeyValueChangedTo: newVersion,
- })
- data.SubVersion = newVersion
-
- // TODO: optimize
- count := 0
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- count++
- return true
- })
- if count > 0 {
- if pc.option.PullThresholdForTopic != -1 {
- newVal := pc.option.PullThresholdForTopic / count
- if newVal == 0 {
- newVal = 1
- }
- rlog.Info("The PullThresholdForTopic is changed", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: pc.option.PullThresholdForTopic,
- rlog.LogKeyValueChangedTo: newVal,
- })
- pc.option.PullThresholdForTopic = newVal
- }
-
- if pc.option.PullThresholdSizeForTopic != -1 {
- newVal := pc.option.PullThresholdSizeForTopic / count
- if newVal == 0 {
- newVal = 1
- }
- rlog.Info("The PullThresholdSizeForTopic is changed", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: pc.option.PullThresholdSizeForTopic,
- rlog.LogKeyValueChangedTo: newVal,
- })
- }
- }
- pc.client.SendHeartbeatToAllBrokerWithLock()
-}
-
-func (pc *pushConsumer) validate() {
- internal.ValidateGroup(pc.consumerGroup)
-
- if pc.consumerGroup == internal.DefaultConsumerGroup {
- // TODO FQA
- rlog.Error(fmt.Sprintf("consumerGroup can't equal [%s], please specify another one.", internal.DefaultConsumerGroup), nil)
- }
-
- if len(pc.subscribedTopic) == 0 {
- rlog.Error("number of subscribed topics is 0.", nil)
- }
-
- if pc.option.ConsumeConcurrentlyMaxSpan < 1 || pc.option.ConsumeConcurrentlyMaxSpan > 65535 {
- if pc.option.ConsumeConcurrentlyMaxSpan == 0 {
- pc.option.ConsumeConcurrentlyMaxSpan = 1000
- } else {
- rlog.Error("option.ConsumeConcurrentlyMaxSpan out of range [1, 65535]", nil)
- }
- }
-
- if pc.option.PullThresholdForQueue < 1 || pc.option.PullThresholdForQueue > 65535 {
- if pc.option.PullThresholdForQueue == 0 {
- pc.option.PullThresholdForQueue = 1024
- } else {
- rlog.Error("option.PullThresholdForQueue out of range [1, 65535]", nil)
- }
- }
-
- if pc.option.PullThresholdForTopic < 1 || pc.option.PullThresholdForTopic > 6553500 {
- if pc.option.PullThresholdForTopic == 0 {
- pc.option.PullThresholdForTopic = 102400
- } else {
- rlog.Error("option.PullThresholdForTopic out of range [1, 6553500]", nil)
- }
- }
-
- if pc.option.PullThresholdSizeForQueue < 1 || pc.option.PullThresholdSizeForQueue > 1024 {
- if pc.option.PullThresholdSizeForQueue == 0 {
- pc.option.PullThresholdSizeForQueue = 512
- } else {
- rlog.Error("option.PullThresholdSizeForQueue out of range [1, 1024]", nil)
- }
- }
-
- if pc.option.PullThresholdSizeForTopic < 1 || pc.option.PullThresholdSizeForTopic > 102400 {
- if pc.option.PullThresholdSizeForTopic == 0 {
- pc.option.PullThresholdSizeForTopic = 51200
- } else {
- rlog.Error("option.PullThresholdSizeForTopic out of range [1, 102400]", nil)
- }
- }
-
- if pc.option.PullInterval < 0 || pc.option.PullInterval > 65535*time.Millisecond {
- rlog.Error("option.PullInterval out of range [0, 65535]", nil)
- }
-
- if pc.option.ConsumeMessageBatchMaxSize < 1 || pc.option.ConsumeMessageBatchMaxSize > 1024 {
- if pc.option.ConsumeMessageBatchMaxSize == 0 {
- pc.option.ConsumeMessageBatchMaxSize = 1
- } else {
- rlog.Error("option.ConsumeMessageBatchMaxSize out of range [1, 1024]", nil)
- }
- }
-
- if pc.option.PullBatchSize < 1 || pc.option.PullBatchSize > 1024 {
- if pc.option.PullBatchSize == 0 {
- pc.option.PullBatchSize = 32
- } else {
- rlog.Error("option.PullBatchSize out of range [1, 1024]", nil)
- }
- }
-}
-
-func (pc *pushConsumer) pullMessage(request *PullRequest) {
- rlog.Debug("start a new Pull Message task for PullRequest", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- var sleepTime time.Duration
- pq := request.pq
- go primitive.WithRecover(func() {
- for {
- select {
- case <-pc.done:
- rlog.Info("push consumer close pullMessage.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- default:
- pc.submitToConsume(request.pq, request.mq)
- if request.pq.IsDroppd() {
- rlog.Info("push consumer quit pullMessage for dropped queue.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- }
- })
-
- for {
- NEXT:
- select {
- case <-pc.done:
- rlog.Info("push consumer close message handle.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- default:
- }
-
- if pq.IsDroppd() {
- rlog.Debug("the request was dropped, so stop task", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- return
- }
- if sleepTime > 0 {
- rlog.Debug(fmt.Sprintf("pull MessageQueue: %d sleep %d ms for mq: %v", request.mq.QueueId, sleepTime/time.Millisecond, request.mq), nil)
- time.Sleep(sleepTime)
- }
- // reset time
- sleepTime = pc.option.PullInterval
- pq.lastPullTime.Store(time.Now())
- err := pc.makeSureStateOK()
- if err != nil {
- rlog.Warning("consumer state error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if pc.pause {
- rlog.Debug(fmt.Sprintf("consumer [%s] of [%s] was paused, execute pull request [%s] later",
- pc.option.InstanceName, pc.consumerGroup, request.String()), nil)
- sleepTime = _PullDelayTimeWhenSuspend
- goto NEXT
- }
-
- cachedMessageSizeInMiB := int(pq.cachedMsgSize / Mb)
- if pq.cachedMsgCount > pc.option.PullThresholdForQueue {
- if pc.queueFlowControlTimes%1000 == 0 {
- rlog.Warning("the cached message count exceeds the threshold, so do flow control", map[string]interface{}{
- "PullThresholdForQueue": pc.option.PullThresholdForQueue,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "count": pq.cachedMsgCount,
- "size(MiB)": cachedMessageSizeInMiB,
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- pc.queueFlowControlTimes++
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
-
- if cachedMessageSizeInMiB > pc.option.PullThresholdSizeForQueue {
- if pc.queueFlowControlTimes%1000 == 0 {
- rlog.Warning("the cached message size exceeds the threshold, so do flow control", map[string]interface{}{
- "PullThresholdSizeForQueue": pc.option.PullThresholdSizeForQueue,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "count": pq.cachedMsgCount,
- "size(MiB)": cachedMessageSizeInMiB,
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- pc.queueFlowControlTimes++
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
-
- if !pc.consumeOrderly {
- if pq.getMaxSpan() > pc.option.ConsumeConcurrentlyMaxSpan {
- if pc.queueMaxSpanFlowControlTimes%1000 == 0 {
- rlog.Warning("the queue's messages span too long, so do flow control", map[string]interface{}{
- "ConsumeConcurrentlyMaxSpan": pc.option.ConsumeConcurrentlyMaxSpan,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "maxSpan": pq.getMaxSpan(),
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
- } else {
- if pq.IsLock() {
- if !request.lockedFirst {
- offset := pc.computePullFromWhere(request.mq)
- brokerBusy := offset < request.nextOffset
- rlog.Info("the first time to pull message, so fix offset from broker, offset maybe changed", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- rlog.LogKeyValueChangedFrom: request.nextOffset,
- rlog.LogKeyValueChangedTo: offset,
- "brokerBusy": brokerBusy,
- })
- if brokerBusy {
- rlog.Info("[NOTIFY_ME] the first time to pull message, but pull request offset larger than "+
- "broker consume offset", map[string]interface{}{"offset": offset})
- }
- request.lockedFirst = true
- request.nextOffset = offset
- }
- } else {
- rlog.Info("pull message later because not locked in broker", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
- }
-
- v, exist := pc.subscriptionDataTable.Load(request.mq.Topic)
- if !exist {
- rlog.Info("find the consumer's subscription failed", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
- beginTime := time.Now()
- var (
- commitOffsetEnable bool
- commitOffsetValue int64
- subExpression string
- )
-
- if pc.model == Clustering {
- commitOffsetValue = pc.storage.read(request.mq, _ReadFromMemory)
- if commitOffsetValue > 0 {
- commitOffsetEnable = true
- }
- }
-
- sd := v.(*internal.SubscriptionData)
- classFilter := sd.ClassFilterMode
- if pc.option.PostSubscriptionWhenPull && classFilter {
- subExpression = sd.SubString
- }
-
- sysFlag := buildSysFlag(commitOffsetEnable, true, subExpression != "", classFilter)
-
- pullRequest := &internal.PullMessageRequestHeader{
- ConsumerGroup: pc.consumerGroup,
- Topic: request.mq.Topic,
- QueueId: int32(request.mq.QueueId),
- QueueOffset: request.nextOffset,
- MaxMsgNums: pc.option.PullBatchSize,
- SysFlag: sysFlag,
- CommitOffset: commitOffsetValue,
- SubExpression: _SubAll,
- ExpressionType: string(TAG),
- SuspendTimeoutMillis: 20 * time.Second,
- }
- //
- //if data.ExpType == string(TAG) {
- // pullRequest.SubVersion = 0
- //} else {
- // pullRequest.SubVersion = data.SubVersion
- //}
-
- brokerResult := pc.defaultConsumer.tryFindBroker(request.mq)
- if brokerResult == nil {
- rlog.Warning("no broker found for mq", map[string]interface{}{
- rlog.LogKeyPullRequest: request.mq.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if brokerResult.Slave {
- pullRequest.SysFlag = clearCommitOffsetFlag(pullRequest.SysFlag)
- }
-
- result, err := pc.client.PullMessage(context.Background(), brokerResult.BrokerAddr, pullRequest)
- if err != nil {
- rlog.Warning("pull message from broker error", map[string]interface{}{
- rlog.LogKeyBroker: brokerResult.BrokerAddr,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if result.Status == primitive.PullBrokerTimeout {
- rlog.Warning("pull broker timeout", map[string]interface{}{
- rlog.LogKeyBroker: brokerResult.BrokerAddr,
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- switch result.Status {
- case primitive.PullFound:
- rlog.Debug(fmt.Sprintf("Topic: %s, QueueId: %d found messages.", request.mq.Topic, request.mq.QueueId), nil)
- prevRequestOffset := request.nextOffset
- request.nextOffset = result.NextBeginOffset
-
- rt := time.Now().Sub(beginTime) / time.Millisecond
- pc.stat.increasePullRT(pc.consumerGroup, request.mq.Topic, int64(rt))
-
- pc.processPullResult(request.mq, result, sd)
-
- msgFounded := result.GetMessageExts()
- firstMsgOffset := int64(math.MaxInt64)
- if msgFounded != nil && len(msgFounded) != 0 {
- firstMsgOffset = msgFounded[0].QueueOffset
- pc.stat.increasePullTPS(pc.consumerGroup, request.mq.Topic, len(msgFounded))
- pq.putMessage(msgFounded...)
- }
- if result.NextBeginOffset < prevRequestOffset || firstMsgOffset < prevRequestOffset {
- rlog.Warning("[BUG] pull message result maybe data wrong", map[string]interface{}{
- "nextBeginOffset": result.NextBeginOffset,
- "firstMsgOffset": firstMsgOffset,
- "prevRequestOffset": prevRequestOffset,
- })
- }
- case primitive.PullNoNewMsg, primitive.PullNoMsgMatched:
- request.nextOffset = result.NextBeginOffset
- pc.correctTagsOffset(request)
- case primitive.PullOffsetIllegal:
- rlog.Warning("the pull request offset illegal", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- "result": result.String(),
- })
- request.nextOffset = result.NextBeginOffset
- pq.WithDropped(true)
- time.Sleep(10 * time.Second)
- pc.storage.update(request.mq, request.nextOffset, false)
- pc.storage.persist([]*primitive.MessageQueue{request.mq})
- pc.processQueueTable.Delete(*request.mq)
- rlog.Warning(fmt.Sprintf("fix the pull request offset: %s", request.String()), nil)
- default:
- rlog.Warning(fmt.Sprintf("unknown pull status: %v", result.Status), nil)
- sleepTime = _PullDelayTimeWhenError
- }
- }
-}
-
-func (pc *pushConsumer) correctTagsOffset(pr *PullRequest) {
- if pr.pq.cachedMsgCount <= 0 {
- pc.storage.update(pr.mq, pr.nextOffset, true)
- }
-}
-
-func (pc *pushConsumer) sendMessageBack(brokerName string, msg *primitive.MessageExt, delayLevel int) bool {
- var brokerAddr string
- if len(brokerName) != 0 {
- brokerAddr = pc.defaultConsumer.client.GetNameSrv().FindBrokerAddrByName(brokerName)
- } else {
- brokerAddr = msg.StoreHost
- }
- _, err := pc.client.InvokeSync(context.Background(), brokerAddr, pc.buildSendBackRequest(msg, delayLevel), 3*time.Second)
- if err != nil {
- return false
- }
- return true
-}
-
-func (pc *pushConsumer) buildSendBackRequest(msg *primitive.MessageExt, delayLevel int) *remote.RemotingCommand {
- req := &internal.ConsumerSendMsgBackRequestHeader{
- Group: pc.consumerGroup,
- OriginTopic: msg.Topic,
- Offset: msg.CommitLogOffset,
- DelayLevel: delayLevel,
- OriginMsgId: msg.MsgId,
- MaxReconsumeTimes: pc.getMaxReconsumeTimes(),
- }
-
- return remote.NewRemotingCommand(internal.ReqConsumerSendMsgBack, req, msg.Body)
-}
-
-func (pc *pushConsumer) suspend() {
- pc.pause = true
- rlog.Info(fmt.Sprintf("suspend consumer: %s", pc.consumerGroup), nil)
-}
-
-func (pc *pushConsumer) resume() {
- pc.pause = false
- pc.doBalance()
- rlog.Info(fmt.Sprintf("resume consumer: %s", pc.consumerGroup), nil)
-}
-
-func (pc *pushConsumer) ResetOffset(topic string, table map[primitive.MessageQueue]int64) {
- //topic := cmd.ExtFields["topic"]
- //group := cmd.ExtFields["group"]
- //if topic == "" || group == "" {
- // rlog.Warning("received reset offset command from: %s, but missing params.", from)
- // return
- //}
- //t, err := strconv.ParseInt(cmd.ExtFields["timestamp"], 10, 64)
- //if err != nil {
- // rlog.Warning("received reset offset command from: %s, but parse time error: %s", err.Error())
- // return
- //}
- //rlog.Infof("invoke reset offset operation from broker. brokerAddr=%s, topic=%s, group=%s, timestamp=%v",
- // from, topic, group, t)
- //
- //offsetTable := make(map[MessageQueue]int64, 0)
- //err = json.Unmarshal(cmd.Body, &offsetTable)
- //if err != nil {
- // rlog.Warning("received reset offset command from: %s, but parse offset table: %s", err.Error())
- // return
- //}
- //v, exist := c.consumerMap.Load(group)
- //if !exist {
- // rlog.Infof("[reset-offset] consumer dose not exist. group=%s", group)
- // return
- //}
- pc.suspend()
- defer pc.resume()
-
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- if _, ok := table[mq]; ok && mq.Topic == topic {
- pq.WithDropped(true)
- pq.clear()
- }
- return true
- })
- time.Sleep(10 * time.Second)
- v, exist := pc.topicSubscribeInfoTable.Load(topic)
- if !exist {
- return
- }
- queuesOfTopic := v.([]*primitive.MessageQueue)
- for _, k := range queuesOfTopic {
- if _, ok := table[*k]; ok {
- pc.storage.update(k, table[*k], false)
- v, exist := pc.processQueueTable.Load(k)
- if !exist {
- continue
- }
- pq := v.(*processQueue)
- pc.removeUnnecessaryMessageQueue(k, pq)
- pc.processQueueTable.Delete(k)
- }
- }
-}
-
-func (pc *pushConsumer) removeUnnecessaryMessageQueue(mq *primitive.MessageQueue, pq *processQueue) bool {
- pc.defaultConsumer.removeUnnecessaryMessageQueue(mq, pq)
- if !pc.consumeOrderly || Clustering != pc.model {
- return true
- }
- // TODO orderly
- return true
-}
-
-func (pc *pushConsumer) consumeInner(ctx context.Context, subMsgs []*primitive.MessageExt) (ConsumeResult, error) {
- if len(subMsgs) == 0 {
- return ConsumeRetryLater, errors.New("msg list empty")
- }
-
- f, exist := pc.consumeFunc.Contains(subMsgs[0].Topic)
-
- // fix lost retry message
- if !exist && strings.HasPrefix(subMsgs[0].Topic, internal.RetryGroupTopicPrefix) {
- f, exist = pc.consumeFunc.Contains(subMsgs[0].GetProperty(primitive.PropertyRetryTopic))
- }
-
- if !exist {
- return ConsumeRetryLater, fmt.Errorf("the consume callback missing for topic: %s", subMsgs[0].Topic)
- }
-
- callback, ok := f.(*PushConsumerCallback)
- if !ok {
- return ConsumeRetryLater, fmt.Errorf("the consume callback assert failed for topic: %s", subMsgs[0].Topic)
- }
- if pc.interceptor == nil {
- return callback.f(ctx, subMsgs...)
- } else {
- var container ConsumeResultHolder
- err := pc.interceptor(ctx, subMsgs, &container, func(ctx context.Context, req, reply interface{}) error {
- msgs := req.([]*primitive.MessageExt)
- r, e := callback.f(ctx, msgs...)
-
- realReply := reply.(*ConsumeResultHolder)
- realReply.ConsumeResult = r
-
- msgCtx, _ := primitive.GetConsumerCtx(ctx)
- msgCtx.Success = realReply.ConsumeResult == ConsumeSuccess
- if realReply.ConsumeResult == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- } else {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- }
- return e
- })
- return container.ConsumeResult, err
- }
-}
-
-// resetRetryAndNamespace modify retry message.
-func (pc *pushConsumer) resetRetryAndNamespace(subMsgs []*primitive.MessageExt) {
- groupTopic := internal.RetryGroupTopicPrefix + pc.consumerGroup
- beginTime := time.Now()
- for idx := range subMsgs {
- msg := subMsgs[idx]
- retryTopic := msg.GetProperty(primitive.PropertyRetryTopic)
- if retryTopic == "" && groupTopic == msg.Topic {
- msg.Topic = retryTopic
- }
- subMsgs[idx].WithProperty(primitive.PropertyConsumeStartTime, strconv.FormatInt(
- beginTime.UnixNano()/int64(time.Millisecond), 10))
- }
-}
-
-func (pc *pushConsumer) consumeMessageCurrently(pq *processQueue, mq *primitive.MessageQueue) {
- msgs := pq.getMessages()
- if msgs == nil {
- return
- }
- for count := 0; count < len(msgs); count++ {
- var subMsgs []*primitive.MessageExt
- if count+pc.option.ConsumeMessageBatchMaxSize > len(msgs) {
- subMsgs = msgs[count:]
- count = len(msgs)
- } else {
- next := count + pc.option.ConsumeMessageBatchMaxSize
- subMsgs = msgs[count:next]
- count = next - 1
- }
- go primitive.WithRecover(func() {
- RETRY:
- if pq.IsDroppd() {
- rlog.Info("the message queue not be able to consume, because it was dropped", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
-
- beginTime := time.Now()
- pc.resetRetryAndNamespace(subMsgs)
- var result ConsumeResult
-
- var err error
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: subMsgs,
- }
- ctx := context.Background()
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
- concurrentCtx := primitive.NewConsumeConcurrentlyContext()
- concurrentCtx.MQ = *mq
- ctx = primitive.WithConcurrentlyCtx(ctx, concurrentCtx)
-
- result, err = pc.consumeInner(ctx, subMsgs)
-
- consumeRT := time.Now().Sub(beginTime)
- if err != nil {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.ExceptionReturn)
- } else if consumeRT >= pc.option.ConsumeTimeout {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.TimeoutReturn)
- } else if result == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- } else if result == ConsumeRetryLater {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- }
-
- pc.stat.increaseConsumeRT(pc.consumerGroup, mq.Topic, int64(consumeRT/time.Millisecond))
-
- if !pq.IsDroppd() {
- msgBackFailed := make([]*primitive.MessageExt, 0)
- if result == ConsumeSuccess {
- pc.stat.increaseConsumeOKTPS(pc.consumerGroup, mq.Topic, len(subMsgs))
- } else {
- pc.stat.increaseConsumeFailedTPS(pc.consumerGroup, mq.Topic, len(subMsgs))
- if pc.model == BroadCasting {
- for i := 0; i < len(subMsgs); i++ {
- rlog.Warning("BROADCASTING, the message consume failed, drop it", map[string]interface{}{
- "message": subMsgs[i],
- })
- }
- } else {
- for i := 0; i < len(subMsgs); i++ {
- msg := subMsgs[i]
- if !pc.sendMessageBack(mq.BrokerName, msg, concurrentCtx.DelayLevelWhenNextConsume) {
- msg.ReconsumeTimes += 1
- msgBackFailed = append(msgBackFailed, msg)
- }
- }
- }
- }
-
- offset := pq.removeMessage(subMsgs...)
-
- if offset >= 0 && !pq.IsDroppd() {
- pc.storage.update(mq, int64(offset), true)
- }
- if len(msgBackFailed) > 0 {
- subMsgs = msgBackFailed
- time.Sleep(5 * time.Second)
- goto RETRY
- }
- } else {
- rlog.Warning("processQueue is dropped without process consume result.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- "message": subMsgs,
- })
- }
- })
- }
-}
-
-func (pc *pushConsumer) consumeMessageOrderly(pq *processQueue, mq *primitive.MessageQueue) {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- return
- }
-
- lock := pc.queueLock.fetchLock(*mq)
- lock.Lock()
- defer lock.Unlock()
- if pc.model == BroadCasting || (pq.IsLock() && !pq.isLockExpired()) {
- beginTime := time.Now()
-
- continueConsume := true
- for continueConsume {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- break
- }
- if pc.model == Clustering {
- if !pq.IsLock() {
- rlog.Warning("the message queue not locked, so consume later", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pc.tryLockLaterAndReconsume(mq, 10)
- return
- }
- if pq.isLockExpired() {
- rlog.Warning("the message queue lock expired, so consume later", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pc.tryLockLaterAndReconsume(mq, 10)
- return
- }
- }
- interval := time.Now().Sub(beginTime)
- if interval > pc.option.MaxTimeConsumeContinuously {
- time.Sleep(10 * time.Millisecond)
- return
- }
- batchSize := pc.option.ConsumeMessageBatchMaxSize
- msgs := pq.takeMessages(batchSize)
-
- pc.resetRetryAndNamespace(msgs)
-
- if len(msgs) == 0 {
- continueConsume = false
- break
- }
-
- // TODO: add message consumer hook
- beginTime = time.Now()
-
- ctx := context.Background()
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: msgs,
- }
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
-
- orderlyCtx := primitive.NewConsumeOrderlyContext()
- orderlyCtx.MQ = *mq
- ctx = primitive.WithOrderlyCtx(ctx, orderlyCtx)
-
- pq.lockConsume.Lock()
- result, _ := pc.consumeInner(ctx, msgs)
- pq.lockConsume.Unlock()
-
- if result == Rollback || result == SuspendCurrentQueueAMoment {
- rlog.Warning("consumeMessage Orderly return not OK", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- "messages": msgs,
- rlog.LogKeyMessageQueue: mq,
- })
- }
-
- // jsut put consumeResult in consumerMessageCtx
- //interval = time.Now().Sub(beginTime)
- //consumeReult := SuccessReturn
- //if interval > pc.option.ConsumeTimeout {
- // consumeReult = TimeoutReturn
- //} else if SuspendCurrentQueueAMoment == result {
- // consumeReult = FailedReturn
- //} else if ConsumeSuccess == result {
- // consumeReult = SuccessReturn
- //}
-
- // process result
- commitOffset := int64(-1)
- if pc.option.AutoCommit {
- switch result {
- case Commit, Rollback:
- rlog.Warning("the message queue consume result is illegal, we think you want to ack these message: %v", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- })
- case ConsumeSuccess:
- commitOffset = pq.commit()
- case SuspendCurrentQueueAMoment:
- if pc.checkReconsumeTimes(msgs) {
- pq.makeMessageToCosumeAgain(msgs...)
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- } else {
- commitOffset = pq.commit()
- }
- default:
- }
- } else {
- switch result {
- case ConsumeSuccess:
- case Commit:
- commitOffset = pq.commit()
- case Rollback:
- // pq.rollback
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- case SuspendCurrentQueueAMoment:
- if pc.checkReconsumeTimes(msgs) {
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- }
- default:
- }
- }
- if commitOffset > 0 && !pq.IsDroppd() {
- _ = pc.updateOffset(mq, commitOffset)
- }
- }
- } else {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- pc.tryLockLaterAndReconsume(mq, 100)
- }
-}
-
-func (pc *pushConsumer) checkReconsumeTimes(msgs []*primitive.MessageExt) bool {
- suspend := false
- if len(msgs) != 0 {
- maxReconsumeTimes := pc.getOrderlyMaxReconsumeTimes()
- for _, msg := range msgs {
- if msg.ReconsumeTimes > maxReconsumeTimes {
- rlog.Warning(fmt.Sprintf("msg will be send to retry topic due to ReconsumeTimes > %d, \n", maxReconsumeTimes), nil)
- msg.WithProperty("RECONSUME_TIME", strconv.Itoa(int(msg.ReconsumeTimes)))
- if !pc.sendMessageBack("", msg, -1) {
- suspend = true
- msg.ReconsumeTimes += 1
- }
- } else {
- suspend = true
- msg.ReconsumeTimes += 1
- }
- }
- }
- return suspend
-}
-
-func (pc *pushConsumer) getOrderlyMaxReconsumeTimes() int32 {
- if pc.option.MaxReconsumeTimes == -1 {
- return math.MaxInt32
- } else {
- return pc.option.MaxReconsumeTimes
- }
-}
-
-func (pc *pushConsumer) getMaxReconsumeTimes() int32 {
- if pc.option.MaxReconsumeTimes == -1 {
- return 16
- } else {
- return pc.option.MaxReconsumeTimes
- }
-}
-
-func (pc *pushConsumer) tryLockLaterAndReconsume(mq *primitive.MessageQueue, delay int64) {
- time.Sleep(time.Duration(delay) * time.Millisecond)
- if pc.lock(mq) == true {
- pc.submitConsumeRequestLater(10)
- } else {
- pc.submitConsumeRequestLater(3000)
- }
-}
-
-func (pc *pushConsumer) submitConsumeRequestLater(suspendTimeMillis int64) {
- if suspendTimeMillis == -1 {
- suspendTimeMillis = int64(pc.option.SuspendCurrentQueueTimeMillis / time.Millisecond)
- }
- if suspendTimeMillis < 10 {
- suspendTimeMillis = 10
- } else if suspendTimeMillis > 30000 {
- suspendTimeMillis = 30000
- }
- time.Sleep(time.Duration(suspendTimeMillis) * time.Millisecond)
-}
diff --git a/consumer/push_consumer_test.go b/consumer/push_consumer_test.go
deleted file mode 100644
index 78bc1f7..0000000
--- a/consumer/push_consumer_test.go
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func mockB4Start(c *pushConsumer) {
- c.topicSubscribeInfoTable.Store("TopicTest", []*primitive.MessageQueue{})
-}
-
-func TestStart(t *testing.T) {
- Convey("test Start method", t, func() {
- c, _ := NewPushConsumer(
- WithGroupName("testGroup"),
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithConsumerModel(BroadCasting),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- client := internal.NewMockRMQClient(ctrl)
- c.client = client
-
- err := c.Subscribe("TopicTest", MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return ConsumeSuccess, nil
- })
-
- _, exists := c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeTrue)
-
- err = c.Unsubscribe("TopicTest")
- So(err, ShouldBeNil)
- _, exists = c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeFalse)
-
- err = c.Subscribe("TopicTest", MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return ConsumeSuccess, nil
- })
-
- _, exists = c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeTrue)
-
- client.EXPECT().ClientID().Return("127.0.0.1@DEFAULT")
- client.EXPECT().Start().Return()
- client.EXPECT().RegisterConsumer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().UpdateTopicRouteInfo().AnyTimes().Return()
-
- Convey("test topic route info not found", func() {
- client.EXPECT().Shutdown().Return()
- err = c.Start()
- So(err.Error(), ShouldContainSubstring, "route info not found")
- })
-
- Convey("test topic route info found", func() {
- client.EXPECT().RebalanceImmediately().Return()
- client.EXPECT().CheckClientInBroker().Return()
- client.EXPECT().SendHeartbeatToAllBrokerWithLock().Return()
- mockB4Start(c)
- err = c.Start()
- So(err, ShouldBeNil)
- })
- })
-}
diff --git a/consumer/statistics.go b/consumer/statistics.go
deleted file mode 100644
index e9d5d79..0000000
--- a/consumer/statistics.go
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "container/list"
- "fmt"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type StatsManager struct {
- startOnce sync.Once
- closeOnce sync.Once
- topicAndGroupConsumeOKTPS *statsItemSet
- topicAndGroupConsumeRT *statsItemSet
- topicAndGroupConsumeFailedTPS *statsItemSet
- topicAndGroupPullTPS *statsItemSet
- topicAndGroupPullRT *statsItemSet
-}
-
-func NewStatsManager() *StatsManager {
- mgr := &StatsManager{}
- mgr.topicAndGroupConsumeOKTPS = newStatsItemSet("CONSUME_OK_TPS")
- mgr.topicAndGroupConsumeRT = newStatsItemSet("CONSUME_RT")
- mgr.topicAndGroupConsumeFailedTPS = newStatsItemSet("CONSUME_FAILED_TPS")
- mgr.topicAndGroupPullTPS = newStatsItemSet("PULL_TPS")
- mgr.topicAndGroupPullRT = newStatsItemSet("PULL_RT")
- return mgr
-}
-
-type ConsumeStatus struct {
- PullRT float64
- PullTPS float64
- ConsumeRT float64
- ConsumeOKTPS float64
- ConsumeFailedTPS float64
- ConsumeFailedMsgs int64
-}
-
-func (mgr *StatsManager) increasePullRT(group, topic string, rt int64) {
- mgr.topicAndGroupPullRT.addValue(topic+"@"+group, rt, 1)
-}
-
-func (mgr *StatsManager) increasePullTPS(group, topic string, msgs int) {
- mgr.topicAndGroupPullTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) increaseConsumeRT(group, topic string, rt int64) {
- mgr.topicAndGroupConsumeRT.addValue(topic+"@"+group, rt, 1)
-}
-
-func (mgr *StatsManager) increaseConsumeOKTPS(group, topic string, msgs int) {
- mgr.topicAndGroupConsumeOKTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) increaseConsumeFailedTPS(group, topic string, msgs int) {
- mgr.topicAndGroupConsumeFailedTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) GetConsumeStatus(group, topic string) ConsumeStatus {
- cs := ConsumeStatus{}
- ss := mgr.getPullRT(group, topic)
- cs.PullTPS = ss.tps
-
- ss = mgr.getPullTPS(group, topic)
- cs.PullTPS = ss.tps
-
- ss = mgr.getConsumeRT(group, topic)
- cs.ConsumeRT = ss.avgpt
-
- ss = mgr.getConsumeOKTPS(group, topic)
- cs.ConsumeOKTPS = ss.tps
-
- ss = mgr.getConsumeFailedTPS(group, topic)
-
- cs.ConsumeFailedTPS = ss.tps
-
- ss = mgr.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + group)
- cs.ConsumeFailedMsgs = ss.sum
- return cs
-}
-
-func (mgr *StatsManager) ShutDownStat() {
- mgr.closeOnce.Do(func() {
- close(mgr.topicAndGroupConsumeOKTPS.closed)
- close(mgr.topicAndGroupConsumeRT.closed)
- close(mgr.topicAndGroupConsumeFailedTPS.closed)
- close(mgr.topicAndGroupPullTPS.closed)
- close(mgr.topicAndGroupPullRT.closed)
- })
-}
-
-func (mgr *StatsManager) getPullRT(group, topic string) statsSnapshot {
- return mgr.topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getPullTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupPullTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getConsumeRT(group, topic string) statsSnapshot {
- ss := mgr.topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group)
- if ss.sum == 0 {
- return mgr.topicAndGroupConsumeRT.getStatsDataInHour(topic + "@" + group)
- }
- return ss
-}
-
-func (mgr *StatsManager) getConsumeOKTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupConsumeOKTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getConsumeFailedTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupConsumeFailedTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-var csListLock sync.Mutex
-
-func computeStatsData(csList *list.List) statsSnapshot {
- csListLock.Lock()
- defer csListLock.Unlock()
- tps, avgpt, sum := 0.0, 0.0, int64(0)
- if csList.Len() > 0 {
- first := csList.Front().Value.(callSnapshot)
- last := csList.Back().Value.(callSnapshot)
- sum = last.value - first.value
- tps = float64(sum*1000.0) / float64(last.timestamp-first.timestamp)
- timesDiff := last.time - first.time
- if timesDiff > 0 {
- avgpt = float64(sum*1.0) / float64(timesDiff)
- }
- }
- return statsSnapshot{
- tps: tps,
- avgpt: avgpt,
- sum: sum,
- }
-}
-
-type statsItemSet struct {
- statsName string
- statsItemTable sync.Map
- closed chan struct{}
-}
-
-func newStatsItemSet(statsName string) *statsItemSet {
- sis := &statsItemSet{
- statsName: statsName,
- closed: make(chan struct{}),
- }
- sis.init()
- return sis
-}
-
-func (sis *statsItemSet) init() {
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(10 * time.Second)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInSeconds()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(10 * time.Minute)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInMinutes()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInHour()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextMinutesTime().Sub(time.Now()))
- ticker := time.NewTicker(time.Minute)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtMinutes()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextHourTime().Sub(time.Now()))
- ticker := time.NewTicker(time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtHour()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextMonthTime().Sub(time.Now()))
- ticker := time.NewTicker(24 * time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtDay()
- }
- }
- })
-}
-
-func (sis *statsItemSet) samplingInSeconds() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInSeconds()
- return true
- })
-}
-
-func (sis *statsItemSet) samplingInMinutes() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInMinutes()
- return true
- })
-}
-
-func (sis *statsItemSet) samplingInHour() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInHour()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtMinutes() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtMinutes()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtHour() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtHour()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtDay() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtDay()
- return true
- })
-}
-
-func (sis *statsItemSet) addValue(key string, incValue, incTimes int64) {
- si := sis.getAndCreateStateItem(key)
- atomic.AddInt64(&si.value, incValue)
- atomic.AddInt64(&si.times, incTimes)
-}
-
-func (sis *statsItemSet) getAndCreateStateItem(key string) *statsItem {
- if val, ok := sis.statsItemTable.Load(key); ok {
- return val.(*statsItem)
- } else {
- si := newStatsItem(sis.statsName, key)
- sis.statsItemTable.Store(key, si)
- return si
- }
-}
-
-func (sis *statsItemSet) getStatsDataInMinute(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInMinute()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsDataInHour(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInHour()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsDataInDay(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInDay()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsItem(key string) *statsItem {
- val, _ := sis.statsItemTable.Load(key)
- return val.(*statsItem)
-}
-
-type statsItem struct {
- value int64
- times int64
- csListMinute *list.List
- csListHour *list.List
- csListDay *list.List
- statsName string
- statsKey string
- csListMinuteLock sync.Mutex
- csListHourLock sync.Mutex
- csListDayLock sync.Mutex
-}
-
-func (si *statsItem) getStatsDataInMinute() statsSnapshot {
- return computeStatsData(si.csListMinute)
-}
-
-func (si *statsItem) getStatsDataInHour() statsSnapshot {
- return computeStatsData(si.csListHour)
-}
-
-func (si *statsItem) getStatsDataInDay() statsSnapshot {
- return computeStatsData(si.csListDay)
-}
-
-func newStatsItem(statsName, statsKey string) *statsItem {
- return &statsItem{
- statsName: statsName,
- statsKey: statsKey,
- csListMinute: list.New(),
- csListHour: list.New(),
- csListDay: list.New(),
- }
-}
-
-func (si *statsItem) samplingInSeconds() {
- si.csListMinuteLock.Lock()
- defer si.csListMinuteLock.Unlock()
- si.csListMinute.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListMinute.Len() > 7 {
- si.csListMinute.Remove(si.csListMinute.Front())
- }
-}
-
-func (si *statsItem) samplingInMinutes() {
- si.csListHourLock.Lock()
- defer si.csListHourLock.Unlock()
- si.csListHour.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListHour.Len() > 7 {
- si.csListHour.Remove(si.csListHour.Front())
- }
-}
-
-func (si *statsItem) samplingInHour() {
- si.csListDayLock.Lock()
- defer si.csListDayLock.Unlock()
- si.csListDay.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListDay.Len() > 25 {
- si.csListHour.Remove(si.csListDay.Front())
- }
-}
-
-func (si *statsItem) printAtMinutes() {
- ss := computeStatsData(si.csListMinute)
- rlog.Info("Stats In One Minute, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func (si *statsItem) printAtHour() {
- ss := computeStatsData(si.csListHour)
- rlog.Info("Stats In One Hour, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func (si *statsItem) printAtDay() {
- ss := computeStatsData(si.csListDay)
- rlog.Info("Stats In One Day, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func nextMinutesTime() time.Time {
- now := time.Now()
- m, _ := time.ParseDuration("1m")
- return now.Add(m)
-}
-
-func nextHourTime() time.Time {
- now := time.Now()
- m, _ := time.ParseDuration("1h")
- return now.Add(m)
-}
-
-func nextMonthTime() time.Time {
- now := time.Now()
- return now.AddDate(0, 1, 0)
-}
-
-type callSnapshot struct {
- timestamp int64
- time int64
- value int64
-}
-
-type statsSnapshot struct {
- sum int64
- tps float64
- avgpt float64
-}
diff --git a/consumer/statistics_test.go b/consumer/statistics_test.go
deleted file mode 100644
index 4836141..0000000
--- a/consumer/statistics_test.go
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "testing"
- "time"
-)
-
-func almostEqual(a, b float64) bool {
- diff := abs(a - b)
- return diff/a < 0.01
-}
-
-func abs(a float64) float64 {
- if a > 0 {
- return a
- }
- return -a
-}
-
-func TestNextMinuteTime(t *testing.T) {
- nextMinute := nextMinutesTime()
- minuteElapse := nextMinute.Sub(time.Now()).Minutes()
- if !almostEqual(minuteElapse, 1.0) {
- t.Errorf("wrong next one minute. want=%f, got=%f", 1.0, minuteElapse)
- }
-}
-
-func TestNextHourTime(t *testing.T) {
- nextHour := nextHourTime()
- hourElapse := nextHour.Sub(time.Now()).Hours()
- if !almostEqual(hourElapse, 1.0) {
- t.Errorf("wrong next one hour. want=%f, got=%f", 1.0, hourElapse)
- }
-}
-
-func TestIncreasePullRTGetPullRT(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
-
- tests := []struct {
- RT int64
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increasePullRT("rocketmq", "default", tt.RT)
- mgr.topicAndGroupPullRT.samplingInSeconds()
- snapshot := mgr.getPullRT("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Pull RT sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-//func TestIncreaseConsumeRTGetConsumeRT(t *testing.T) {
-// ShutDownStat()
-// tests := []struct {
-// RT int64
-// ExpectSum int64
-// }{
-// {1, 0},
-// {1, 1},
-// {1, 2},
-// {1, 3},
-// {1, 4},
-// {1, 5},
-// {1, 6},
-// {1, 6},
-// }
-// for _, tt := range tests {
-// increaseConsumeRT("rocketmq", "default", tt.RT)
-// topicAndGroupConsumeRT.samplingInMinutes()
-// snapshot := getConsumeRT("rocketmq", "default")
-// if snapshot.sum != tt.ExpectSum {
-// t.Errorf("wrong consume RT sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
-// }
-// }
-//}
-
-func TestIncreasePullTPSGetPullTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increasePullTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupPullTPS.samplingInSeconds()
- snapshot := mgr.getPullTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Pull TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestIncreaseConsumeOKTPSGetConsumeOKTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increaseConsumeOKTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupConsumeOKTPS.samplingInSeconds()
- snapshot := mgr.getConsumeOKTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Consume OK TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestIncreaseConsumeFailedTPSGetConsumeFailedTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increaseConsumeFailedTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupConsumeFailedTPS.samplingInSeconds()
- snapshot := mgr.getConsumeFailedTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Consume Failed TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestGetConsumeStatus(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- group, topic := "rocketmq", "default"
-
- tests := []struct {
- RT int
- ExpectFailMessage int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- }
- for _, tt := range tests {
- mgr.increasePullRT(group, topic, int64(tt.RT))
- mgr.increasePullTPS(group, topic, tt.RT)
- mgr.increaseConsumeRT(group, topic, int64(tt.RT))
- mgr.increaseConsumeOKTPS(group, topic, tt.RT)
- mgr.increaseConsumeFailedTPS(group, topic, tt.RT)
- mgr.topicAndGroupPullRT.samplingInSeconds()
- mgr.topicAndGroupPullTPS.samplingInSeconds()
- mgr.topicAndGroupConsumeRT.samplingInMinutes()
- mgr.topicAndGroupConsumeOKTPS.samplingInSeconds()
- mgr.topicAndGroupConsumeFailedTPS.samplingInMinutes()
- status := mgr.GetConsumeStatus(group, topic)
- if status.ConsumeFailedMsgs != tt.ExpectFailMessage {
- t.Errorf("wrong ConsumeFailedMsg. want=0, got=%d", status.ConsumeFailedMsgs)
- }
- }
-}
diff --git a/consumer/strategy.go b/consumer/strategy.go
deleted file mode 100644
index 4a07928..0000000
--- a/consumer/strategy.go
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "strings"
-
- "github.com/stathat/consistent"
-
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-// Strategy Algorithm for message allocating between consumers
-// An allocate strategy proxy for based on machine room nearside priority. An actual allocate strategy can be
-// specified.
-//
-// If any consumer is alive in a machine room, the message queue of the broker which is deployed in the same machine
-// should only be allocated to those. Otherwise, those message queues can be shared along all consumers since there are
-// no alive consumer to monopolize them.
-//
-// Average Hashing queue algorithm
-// Cycle average Hashing queue algorithm
-// Use Message QueueID specified
-// Computer room Hashing queue algorithm, such as Alipay logic room
-// Consistent Hashing queue algorithm
-
-type AllocateStrategy func(string, string, []*primitive.MessageQueue, []string) []*primitive.MessageQueue
-
-func AllocateByAveragely(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- mqSize := len(mqAll)
- cidSize := len(cidAll)
- mod := mqSize % cidSize
-
- var averageSize int
- if mqSize <= cidSize {
- averageSize = 1
- } else {
- if mod > 0 && index < mod {
- averageSize = mqSize/cidSize + 1
- } else {
- averageSize = mqSize / cidSize
- }
- }
-
- var startIndex int
- if mod > 0 && index < mod {
- startIndex = index * averageSize
- } else {
- startIndex = index*averageSize + mod
- }
-
- num := utils.MinInt(averageSize, mqSize-startIndex)
- result := make([]*primitive.MessageQueue, 0)
- for i := 0; i < num; i++ {
- result = append(result, mqAll[(startIndex+i)%mqSize])
- }
- return result
-}
-
-func AllocateByAveragelyCircle(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- result := make([]*primitive.MessageQueue, 0)
- for i := index; i < len(mqAll); i++ {
- if i%len(cidAll) == index {
- result = append(result, mqAll[i])
- }
- }
- return result
-}
-
-// TODO
-func AllocateByMachineNearby(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- return AllocateByAveragely(consumerGroup, currentCID, mqAll, cidAll)
-}
-
-func AllocateByConfig(list []*primitive.MessageQueue) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- return list
- }
-}
-
-func AllocateByMachineRoom(consumeridcs []string) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- var premqAll []*primitive.MessageQueue
- for _, mq := range mqAll {
- temp := strings.Split(mq.BrokerName, "@")
- if len(temp) == 2 {
- for _, idc := range consumeridcs {
- if idc == temp[0] {
- premqAll = append(premqAll, mq)
- }
- }
- }
- }
-
- mod := len(premqAll) / len(cidAll)
- rem := len(premqAll) % len(cidAll)
- startIndex := mod * index
- endIndex := startIndex + mod
-
- result := make([]*primitive.MessageQueue, 0)
- for i := startIndex; i < endIndex; i++ {
- result = append(result, mqAll[i])
- }
- if rem > index {
- result = append(result, premqAll[index+mod*len(cidAll)])
- }
- return result
- }
-}
-
-func AllocateByConsistentHash(virtualNodeCnt int) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- c := consistent.New()
- c.NumberOfReplicas = virtualNodeCnt
- for _, cid := range cidAll {
- c.Add(cid)
- }
-
- result := make([]*primitive.MessageQueue, 0)
- for _, mq := range mqAll {
- clientNode, err := c.Get(mq.String())
- if err != nil {
- rlog.Warning("[BUG] AllocateByConsistentHash err: %s", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- }
- if currentCID == clientNode {
- result = append(result, mq)
- }
- }
- return result
- }
-}
diff --git a/consumer/strategy_test.go b/consumer/strategy_test.go
deleted file mode 100644
index d521b4b..0000000
--- a/consumer/strategy_test.go
+++ /dev/null
@@ -1,487 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func TestAllocateByAveragely(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- Convey("When params is empty", func() {
- result := AllocateByAveragely("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragely("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragely("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 5,
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByAveragely should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := AllocateByAveragely("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByAveragelyCircle(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- Convey("When params is empty", func() {
- result := AllocateByAveragelyCircle("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragelyCircle("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragelyCircle("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 4,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- },
- {
- QueueId: 4,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- },
- {
- QueueId: 5,
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByAveragelyCircle should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := AllocateByAveragelyCircle("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByConfig(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- strategy := AllocateByConfig(queues)
- result := strategy("testGroup", "192.168.24.1@default", queues, []string{"192.168.24.1@default", "192.168.24.2@default"})
- So(result, ShouldResemble, queues)
- })
-}
-
-func TestAllocateByMachineRoom(t *testing.T) {
- Convey("Given some consumer IDCs with a starting value", t, func() {
- idcs := []string{"192.168.24.1", "192.168.24.2"}
- strategy := AllocateByMachineRoom(idcs)
-
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 2,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 5,
- BrokerName: "192.168.24.3@defaultName",
- },
- }
-
- Convey("When params is empty", func() {
- result := strategy("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByMachineRoom should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := strategy("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByConsistentHash(t *testing.T) {
- Convey("Given virtualNodeCnt with a starting value", t, func() {
- virtualNodeCnt := 10
- strategy := AllocateByConsistentHash(virtualNodeCnt)
-
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 2,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 5,
- BrokerName: "192.168.24.3@defaultName",
- },
- }
-
- Convey("When params is empty", func() {
- result := strategy("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.3@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- },
- }
-
- Convey("observe the result of AllocateByMachineRoom", func() {
- for _, value := range cases {
- result := strategy("testGroup", value.currentCid, value.mqAll, value.cidAll)
- rlog.Info("Result Of AllocateByMachineRoom", map[string]interface{}{
- "currentCid": value.currentCid,
- "cidAll": value.cidAll,
- "allocateResult": result,
- })
- }
- })
- })
-}
diff --git a/docs/Introduction.md b/docs/Introduction.md
deleted file mode 100644
index 011603a..0000000
--- a/docs/Introduction.md
+++ /dev/null
@@ -1,130 +0,0 @@
-## How to use
-
-### go mod
-```
-require (
- github.com/apache/rocketmq-client-go/v2 v2.1.0-rc3
-)
-```
-
-### Set Logger
-Go Client define the `Logger` interface for log output, user can specify implementation of private.
-in default, client use `logrus`.
-```
-rlog.SetLogger(Logger)
-```
-
-### Send message
-#### Interface
-```
-Producer interface {
- Start() error
- Shutdown() error
- SendSync(context.Context, *primitive.Message) (*internal.SendResult, error)
- SendOneWay(context.Context, *primitive.Message) error
-}
-```
-
-#### Examples
-- create a new `Producer` instance
-```
-p, err := rocketmq.NewProducer(
- producer.WithNameServer(endPoint),
- //producer.WithNsResolver(primitive.NewPassthroughResolver(endPoint)),
- producer.WithRetry(2),
- producer.WithGroupName("GID_xxxxxx"),
- )
-```
-
-- start the producer
-```go
-err := p.Start()
-```
-
-- send message with sync
-```
-result, err := p.SendSync(context.Background(), &primitive.Message{
- Topic: "test",
- Body: []byte("Hello RocketMQ Go Client!"),
-})
-
-// do something with result
-```
-
-- or send message with oneway
-```
-err := p.SendOneWay(context.Background(), &primitive.Message{
- Topic: "test",
- Body: []byte("Hello RocketMQ Go Client!"),
-})
-```
-Full examples: [producer](../examples/producer)
-
-### Consume Message
-now only support `PushConsumer`
-
-#### Interface
-```
-PushConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be sync to broker before process exit
- Shutdown() error
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (consumer.ConsumeResult, error)) error
-}
-```
-
-#### Usage
-- Create a `PushConsumer` instance
-```
-c, err := rocketmq.NewPushConsumer(
- consumer.WithNameServer(endPoint),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithGroupName("GID_XXXXXX"),
- )
-```
-
-- Subscribe a topic(only support one topic now), and define your consuming function
-```
-err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx *consumer.ConsumeMessageContext,
- msgs []*primitive.MessageExt) (consumer.ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return consumer.ConsumeSuccess, nil
-})
-```
-- start the consumer(**NOTE: MUST after subscribe**)
-```
-err = c.Start()
-```
-
-Full examples: [consumer](../examples/consumer)
-
-
-### Admin: Topic Operation
-
-#### Examples
-- create topic
-```
-testAdmin, err := admin.NewAdmin(admin.WithResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})))
-err = testAdmin.CreateTopic(
- context.Background(),
- admin.WithTopicCreate("newTopic"),
- admin.WithBrokerAddrCreate("127.0.0.1:10911"),
-)
-```
-
-- delete topic
-`ClusterName` not supported yet
-```
-err = testAdmin.DeleteTopic(
- context.Background(),
- admin.WithTopicDelete("newTopic"),
- //admin.WithBrokerAddrDelete("127.0.0.1:10911"), //optional
- //admin.WithNameSrvAddr(nameSrvAddr), //optional
-)
-```
\ No newline at end of file
diff --git a/docs/client-design.gliffy b/docs/client-design.gliffy
deleted file mode 100644
index 418f4d8..0000000
--- a/docs/client-design.gliffy
+++ /dev/null
@@ -1 +0,0 @@
-{"contentType":"application/gliffy+json","version":"1.1","metadata":{"title":"untitled","revision":0,"exportBorder":false},"embeddedResources":{"index":0,"resources":[]},"stage":{"objects":[{"x":31,"y":44,"rotation":0,"id":57,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":27,"lockAspectRatio":false,"lockShape":false,"order":57,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\ [...]
\ No newline at end of file
diff --git a/docs/feature.md b/docs/feature.md
deleted file mode 100644
index eed3e17..0000000
--- a/docs/feature.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# Feature
-
-## Producer
-
-### MessageType
-- [x] NormalMessage
-- [ ] TransactionMessage
-- [ ] DelayMessage
-
-### SendWith
-- [x] Sync
-- [ ] Async
-- [x] OneWay
-
-### Other
-- [ ] Config
-- [ ] MessageId Generate
-- [ ] CompressMsg
-- [ ] LoadBalance
-- [ ] DefaultTopic
-- [ ] VipChannel
-- [ ] Retry
-- [ ] Hook
-- [ ] CheckRequestQueue
-- [ ] MQFaultStrategy
-
-## Consumer
-
-### ReceiveType
-- [x] Push
-- [ ] Pull
-
-### ConsumingType
-- [x] Concurrently
-- [ ] Orderly
-
-### MessageModel
-- [x] CLUSTERING
-- [x] BROADCASTING
-
-### AllocateMessageQueueStrategy
-- [x] AllocateMessageQueueAveragely
-- [x] AllocateMessageQueueAveragelyByCircle
-- [X] AllocateMessageQueueByConfig
-- [X] AllocateMessageQueueByMachineRoom
-
-### Other
-- [x] Rebalance
-- [x] Flow Control
-- [ ] compress
-- [x] ConsumeFromWhere
-- [ ] Retry(sendMessageBack)
-- [ ] Hook
-
-## Common
-- [ ] PollNameServer
-- [x] Heartbeat
-- [x] UpdateTopicRouteInfoFromNameServer
-- [ ] CleanOfflineBroker
-- [ ] ClearExpiredMessage(form consumer consumeMessageService)
-
-## Remoting
-- [x] API
- - [x] InvokeSync
- - [x] InvokeAsync
- - [x] InvokeOneWay
-- [x] Serialize
- - [x] JSON
- - [x] ROCKETMQ
-- [ ] Other
- - [ ] VIPChannel
- - [ ] RPCHook
-
-## Admin
-
-### Topic/Cluster
-- [x] updateTopic
-- [x] deleteTopic
-- [ ] updateSubGroup
-- [ ] deleteSubGroup
-- [ ] updateBrokerConfig
-- [ ] updateTopicPerm
-- [ ] listTopic
-- [ ] topicRoute
-- [ ] topicStatus
-- [ ] topicClusterList
\ No newline at end of file
diff --git a/docs/images/client-design.png b/docs/images/client-design.png
deleted file mode 100644
index 97b86ef..0000000
Binary files a/docs/images/client-design.png and /dev/null differ
diff --git a/docs/zh/native-design_zh.md b/docs/zh/native-design_zh.md
deleted file mode 100644
index dd407a4..0000000
--- a/docs/zh/native-design_zh.md
+++ /dev/null
@@ -1,114 +0,0 @@
-# RocketMQ Go Client Design Draft
-
-## Architecture
-
-### Overview
-
-
-### Description
-在RocketMQ Java Client的实现里面，代码耦合了大量的admin方面的功能， 其为了尽可能的提高代码复用率，代码的依赖关系较为复杂、接口的设计比
-较重、语义的界限不够清晰。因此，为了避免简单的对Java代码进行翻译，故Go客户端进行了重新的设计，剥离了admin相关的逻辑。整体如上面的图所示，在逻辑层次上，按照请求
-顺序，从下到上总共分为三层：
-- remote层：client网络通信和私有协议层，将到每个到服务端(NameServer或Broker)的连接实体抽象为一个client结构，并在这里实现了网络数据的
-序列/反序列化。
-- 公共层：由于remote层对上层只暴露了`Sync/Async/Oneway`三个接口，所以对于特定的Request/Response、连接的管理、路由等信息的处理和维护、
-其它`producer/consumer`共用的逻辑，均在这里实现。
-- 业务逻辑层：在这里实现各种producer、consumer的语义。所有producer、consumer专有的逻辑，均放到这里实现，如`queue selector`,
-`consume balance`, `offset storage`等。除了基础的数据结构外，producer和consumer之间内部的代码不能进行复用。
-
-
-## 设计目标
-0. 兼容cgo版本已经暴露出去的API
-1. 实现语义清晰、轻量的API接口
-2. 依赖关系清晰简单，设计和实现要正交
-
-## 目录结构
-### 源码
-- producer：producer相关逻辑
-- consumer：consumer相关逻辑
-- common（可改名）：连接管理和路由管理相关的通用逻辑
-- remote：网络通信和序列化
-
-### 其它
-- benchmark：压力测试相关代码
-- core：1.2版本cgo的代码库，该目录下的代码将会被移除，只保留API进行兼容，并会被标记为`Deprecated`
-- docs：文档，包括面向用户和开发者
-- examples：示例代码
-- test：集成测试代码
-
-## API
-
-### remote
-```go
-NewRemotingCommand(code int16, header CustomHeader) *RemotingCommand
-
-// send a request to servers and return until response received.
-SendMessageSync(ctx context.Context, brokerAddrs, brokerName string, request *SendMessageRequest, msgs []*Message) (*SendResult, error)
-
-SendMessageAsync(ctx context.Context, brokerAddrs, brokerName string, request *SendMessageRequest, msgs []*Message, f func(result *SendResult)) error
-
-SendMessageOneWay(ctx context.Context, brokerAddrs string, request *SendMessageRequest, msgs []*Message) (*SendResult, error)
-```
-
-### common
-All struct needed has been defined in codebase.
-
-```go
-// PullMessage with sync
-SendMessage(topic string, msgs *[]Message) error
-
-// SendMessageAsync send message with batch by async
-SendMessageAsync(topic string, msgs *[]Message, f func(result *SendResult)) error
-
-// PullMessage with sync
-PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequest) (*PullResult, error)
-
-// PullMessageAsync pull message async
-func PullMessageAsync(ctx context.Context, brokerAddrs string, request *PullMessageRequest, f func(result *PullResult)) error
-
-// QueryMaxOffset with specific queueId and topic
-QueryMaxOffset(topic string, queueId int) error
-
-// QueryConsumerOffset with specific queueId and topic of consumerGroup
-QueryConsumerOffset(consumerGroup, topic string, queue int) (int64, error)
-
-// SearchOffsetByTimestamp with specific queueId and topic
-SearchOffsetByTimestamp(topic string, queue int, timestamp int64) (int64, error)
-
-// UpdateConsumerOffset with specific queueId and topic
-UpdateConsumerOffset(consumerGroup, topic string, queue int, offset int64) error
-```
-
-## Road map
-for more details about features: [feature-list](../feature.md)
-
-### Milestone1(due: 2019.3.10)
-
-#### producer
-- [ ] normal message
-- [ ] order message
-
-#### consumer
-- [ ] normal message with pull/push
-- [ ] order message with pull/push
-- [ ] rebalance
-- [ ] offset manager
-
-#### common
-- [ ] API wrapper
-- [ ] connections manager
-- [ ] route
-
-#### remote
-- [ ] serializer
-- [ ] communication
-- [ ] processor
-- [ ] RPC
-
-### Milestone2 (2019.4.12)
-- Transaction Message
-- ACL
-- Message Tracing
-
-## sub project
-- RocketMQ Administration tools: JVM too heavy for command line tools
\ No newline at end of file
diff --git a/docs/zh/rocketmq-protocol_zh.md b/docs/zh/rocketmq-protocol_zh.md
deleted file mode 100644
index 818d2e1..0000000
--- a/docs/zh/rocketmq-protocol_zh.md
+++ /dev/null
@@ -1,117 +0,0 @@
-# RocketMQ 通信协议
-
-在 RocketMQ 中，`RemotingCommand` 是 RocketMQ 通信的基本对象，Request/Response 最后均被包装成 `RemotingCommand`。一个 `RemotingCommand` 在被序列化后的格式如下：
-
-```
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ frame_size | header_length | header_body | body +
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ 4bytes | 4bytes | (21 + r_len + e_len) bytes | remain bytes +
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-```
-
-| item | type | description |
-| :-: | :-: | :-: |
-| frame_size | `int32` | 一个 `RemotingCommand` 数据包大小 |
-| header_length | `int32` | 高8位表示数据的序列化方式，余下的表示真实 header 长度 |
-| header_body | `[]byte` | header 的 payload，长度由附带的 `remark` 和 `properties` 决定|
-| body | `[]byte` | 具体 Request/Response 的 payload |
-
-## Header
-
-RocketMQ 的 Header 序列化方式有两种：JSON 和 RocketMQ 私有的序列化方式。JSON 序列化方式不再赘述。具体可以参考 Java `RemotingCommand` 类。
-
-主要介绍 RocketMQ 的私有序列化方式。
-
-在序列化的时候，需要将序列化方式记录进数据包里面，即对 `header_length` 进行编码
-
-```go
-// 编码算法
-
-// 编码后的 header_length
-var header_length int32
-
-// 实际的 header 长度
-var headerDataLen int32
-
-// 序列化方式
-var SerializedType byte
-
-result := make([]byte, 4)
-result[0]|SerializedType
-result[1]|byte((headerDataLen >> 16) & 0xFF)
-result[2]|byte((headerDataLen >> 8) & 0xFF)
-result[3]|byte(headerDataLen & 0xFF)
-binary.Read(result, binary.BigEndian, &header_length)
-
-// 解码算法
-headerDataLen := header_length & 0xFFFFFF
-SerializedType := byte((header_length >> 24) & 0xFF)
-```
-
-### Header Frame
-
-```
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ request_code | l_flag | v_flag | opaque | request_flag | r_len | r_body | e_len | e_body +
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ 2bytes | 1byte | 2bytes | 4bytes | 4 bytes | 4 bytes | r_len bytes | 4 bytes | e_len bytes +
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-```
-
-| item | type | description |
-| :-: | :-: | :-: |
-| request_code | `int16` | 哪一种 Request 或 ResponseCode，具体类别由 request_flag 决定 |
-| l_flag | `byte` | language 位，用来标识Request来源方的开发语言 |
-| v_flag | `int16` | 版本标记位 |
-| request_flag |`int32`| Header标记位，用来标记该 `RemotingCommand` 的类型和请求方式 |
-| opaque | `int32` | 标识 Request/Response 的 RequestID，Broker 返回的 Response 通过该值和 Client 缓存的 Request 一一对应 |
-| r_len | `int32` | length of remark, remark 是 Request/Response 的附带说明信息，一般在 Response 中用来说明具体的错误原因 |
-| r_body | `[]byte` | payload of remark |
-| e_len | `int32` | length of extended fields，即 properties，一些非标准字段会存储在这里，在 RocketMQ 的各种 feature 中均有广泛应用 |
-| e_body | `int32` | payload of extended fields |
-
-## Body
-
-`body` 是具体的 Request/Response 的数据，在 RocketMQ 中，有许多种 Request/Response。每个类有自己的序列化和反序列方式，由于种类过多，
-这里就不再展开。可以具体参考Java代码中对`CommandCustomHeader`的使用。下面列一些 Client 使用到的 Request 和 Response。
-
-### RequestCode
-
-| item | type | description |
-| :-: | :-: | :-: |
-| SEND_MESSAGE | 10 | 向broker发送消息 |
-| PULL_MESSAGE | 11 | 从broker拉取消息，client的push模式也是通过pull的长轮询来实现的 |
-| TODO... | | |
-
-### ResponseCode
-
-| item | type | description |
-| :-: | :-: | :-: |
-| FLUSH_DISK_TIMEOUT | 10 | broker 存储层刷盘超时 |
-| SLAVE_NOT_AVAILABLE | 11 | slave 节点无法服务 |
-| FLUSH_SLAVE_TIMEOUT | 12 | 数据同步到 slave 超时 |
-| MESSAGE_ILLEGAL | 13 | 消息格式不合格 |
-| SERVICE_NOT_AVAILABLE | 14 | broker 暂时不可用 |
-| VERSION_NOT_SUPPORTED | 15 | 不支持的请求，目前没有看到使用 |
-| NO_PERMISSION | 16 | 对 broker、topic 或 subscription 无访问权限 |
-| TOPIC_EXIST_ALREADY | 18 | topic 已存在，目前没看到使用 |
-| PULL_NOT_FOUND | 19 | 没拉到消息，大多为 offset 错误 |
-| PULL_RETRY_IMMEDIATELY | 20 | 建议 client 立即重新拉取消息 |
-| PULL_OFFSET_MOVED | 21 | offset 太小或太大 |
-| QUERY_NOT_FOUND | 22 | 管理面 Response，TODO |
-| SUBSCRIPTION_PARSE_FAILED | 23 | 订阅数据解析失败 |
-| SUBSCRIPTION_NOT_EXIST | 24 | 订阅不存在 |
-| SUBSCRIPTION_NOT_LATEST | 25 | 订阅数据版本和 request 数据版本不匹配 |
-| SUBSCRIPTION_GROUP_NOT_EXIST | 26 | 订阅组不存在 |
-| FILTER_DATA_NOT_EXIST | 27 | filter 数据不存在 |
-| FILTER_DATA_NOT_LATEST | 28 | filter 数据版本和 request 数据版本不匹配 |
-| TRANSACTION_SHOULD_COMMIT | 200 | 事务 Response，TODO |
-| TRANSACTION_SHOULD_ROLLBACK | 201 | 事务 Response，TODO |
-| TRANSACTION_STATE_UNKNOW | 202 | 事务 Response，TODO | |
-| TRANSACTION_STATE_GROUP_WRONG | 203 | 事务 Response，TODO |
-| NO_BUYER_ID | 204 | 不知道是什么，没看到 broker 端在使用 |
-| NOT_IN_CURRENT_UNIT | 205 | 不知道是什么，没看到 broker 端在使用 |
-| CONSUMER_NOT_ONLINE | 206 | consumer 不在线，控制面 response |
-| CONSUME_MSG_TIMEOUT | 207 | client request 等待 broker 相应超时 |
-| NO_MESSAGE | 208 | 控制面 response，由 client 自己设置，不清楚具体用途 |
diff --git a/errors/errors.go b/errors/errors.go
deleted file mode 100644
index 195984e..0000000
--- a/errors/errors.go
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package errors
-
-import "errors"
-
-var (
- ErrRequestTimeout = errors.New("request timeout")
- ErrMQEmpty = errors.New("MessageQueue is nil")
- ErrOffset = errors.New("offset < 0")
- ErrNumbers = errors.New("numbers < 0")
- ErrEmptyTopic = errors.New("empty topic")
- ErrEmptyNameSrv = errors.New("empty namesrv")
- ErrEmptyGroupID = errors.New("empty group id")
- ErrTestMin = errors.New("test minutes must be positive integer")
- ErrOperationInterval = errors.New("operation interval must be positive integer")
- ErrMessageBody = errors.New("message body size must be positive integer")
- ErrEmptyExpression = errors.New("empty expression")
- ErrCreated = errors.New("consumer group has been created")
- ErrBrokerNotFound = errors.New("broker can not found")
- ErrStartTopic = errors.New("cannot subscribe topic since client either failed to start or has been shutdown.")
- ErrResponse = errors.New("response error")
- ErrCompressLevel = errors.New("unsupported compress level")
- ErrUnknownIP = errors.New("unknown IP address")
- ErrService = errors.New("service close is not running, please check")
- ErrTopicNotExist = errors.New("topic not exist")
- ErrNotExisted = errors.New("not existed")
- ErrNoNameserver = errors.New("nameServerAddrs can't be empty.")
- ErrMultiIP = errors.New("multiple IP addr does not support")
- ErrIllegalIP = errors.New("IP addr error")
- ErrTopicEmpty = errors.New("topic is nil")
- ErrMessageEmpty = errors.New("message is nil")
- ErrNotRunning = errors.New("producer not started")
- ErrPullConsumer = errors.New("pull consumer has not supported")
- ErrMultipleTopics = errors.New("the topic of the messages in one batch should be the same")
-)
diff --git a/examples/admin/topic/main.go b/examples/admin/topic/main.go
deleted file mode 100644
index ef9a536..0000000
--- a/examples/admin/topic/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
-
- "github.com/apache/rocketmq-client-go/v2/admin"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- topic := "newOne"
- //clusterName := "DefaultCluster"
- nameSrvAddr := []string{"127.0.0.1:9876"}
- brokerAddr := "127.0.0.1:10911"
-
- testAdmin, err := admin.NewAdmin(admin.WithResolver(primitive.NewPassthroughResolver(nameSrvAddr)))
- if err != nil {
- fmt.Println(err.Error())
- }
-
- //create topic
- err = testAdmin.CreateTopic(
- context.Background(),
- admin.WithTopicCreate(topic),
- admin.WithBrokerAddrCreate(brokerAddr),
- )
- if err != nil {
- fmt.Println("Create topic error:", err.Error())
- }
-
- //deletetopic
- err = testAdmin.DeleteTopic(
- context.Background(),
- admin.WithTopicDelete(topic),
- //admin.WithBrokerAddrDelete(brokerAddr),
- //admin.WithNameSrvAddr(nameSrvAddr),
- )
- if err != nil {
- fmt.Println("Delete topic error:", err.Error())
- }
-
- err = testAdmin.Close()
- if err != nil {
- fmt.Printf("Shutdown admin error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/acl/main.go b/examples/consumer/acl/main.go
deleted file mode 100644
index 96d0dd0..0000000
--- a/examples/consumer/acl/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- )
- if err != nil {
- fmt.Println("init consumer error: " + err.Error())
- os.Exit(0)
- }
-
- err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/broadcast/main.go b/examples/consumer/broadcast/main.go
deleted file mode 100644
index 219ab3e..0000000
--- a/examples/consumer/broadcast/main.go
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerModel(consumer.BroadCasting),
- )
- err := c.Subscribe("min", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/delay/main.go b/examples/consumer/delay/main.go
deleted file mode 100644
index 1bbe6ed..0000000
--- a/examples/consumer/delay/main.go
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
-
- for _, msg := range msgs {
- t := time.Now().UnixNano()/int64(time.Millisecond) - msg.BornTimestamp
- fmt.Printf("Receive message[msgId=%s] %d ms later\n", msg.MsgId, t)
- }
-
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/interceptor/main.go b/examples/consumer/interceptor/main.go
deleted file mode 100644
index 83f43b0..0000000
--- a/examples/consumer/interceptor/main.go
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithInterceptor(UserFistInterceptor(), UserSecondInterceptor()))
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
-
-func UserFistInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- msgCtx, _ := primitive.GetConsumerCtx(ctx)
- fmt.Printf("msgCtx: %v, mehtod: %s", msgCtx, primitive.GetMethod(ctx))
-
- msgs := req.([]*primitive.MessageExt)
- fmt.Printf("user first interceptor before invoke: %v\n", msgs)
- e := next(ctx, msgs, reply)
-
- holder := reply.(*consumer.ConsumeResultHolder)
- fmt.Printf("user first interceptor after invoke: %v, result: %v\n", msgs, holder)
- return e
- }
-}
-
-func UserSecondInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- msgs := req.([]*primitive.MessageExt)
- fmt.Printf("user second interceptor before invoke: %v\n", msgs)
- e := next(ctx, msgs, reply)
- holder := reply.(*consumer.ConsumeResultHolder)
- fmt.Printf("user second interceptor after invoke: %v, result: %v\n", msgs, holder)
- return e
- }
-}
diff --git a/examples/consumer/namespace/main.go b/examples/consumer/namespace/main.go
deleted file mode 100644
index d46f210..0000000
--- a/examples/consumer/namespace/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- consumer.WithNamespace("namespace"),
- )
- if err != nil {
- fmt.Println("init consumer error: " + err.Error())
- os.Exit(0)
- }
-
- err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/orderly/main.go b/examples/consumer/orderly/main.go
deleted file mode 100644
index 0f33e5f..0000000
--- a/examples/consumer/orderly/main.go
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerOrder(true),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
- fmt.Printf("orderly context: %v\n", orderlyCtx)
- fmt.Printf("subscribe orderly callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/pull/main.go b/examples/consumer/pull/main.go
deleted file mode 100644
index 5b5819e..0000000
--- a/examples/consumer/pull/main.go
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-func main() {
- c, err := rocketmq.NewPullConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- if err != nil {
- rlog.Fatal(fmt.Sprintf("fail to new pullConsumer: %s", err), nil)
- }
- err = c.Start()
- if err != nil {
- rlog.Fatal(fmt.Sprintf("fail to new pullConsumer: %s", err), nil)
- }
-
- ctx := context.Background()
- queue := primitive.MessageQueue{
- Topic: "TopicTest",
- BrokerName: "", // replace with your broker name. otherwise, pull will failed.
- QueueId: 0,
- }
-
- offset := int64(0)
- for {
- resp, err := c.PullFrom(ctx, queue, offset, 10)
- if err != nil {
- if err == errors.ErrRequestTimeout {
- fmt.Printf("timeout \n")
- time.Sleep(1 * time.Second)
- continue
- }
- fmt.Printf("unexpectable err: %v \n", err)
- return
- }
- if resp.Status == primitive.PullFound {
- fmt.Printf("pull message success. nextOffset: %d \n", resp.NextBeginOffset)
- for _, msg := range resp.GetMessageExts() {
- fmt.Printf("pull msg: %v \n", msg)
- }
- }
- offset = resp.NextBeginOffset
- }
-}
diff --git a/examples/consumer/retry/concurrent/main.go b/examples/consumer/retry/concurrent/main.go
deleted file mode 100644
index 49f16d1..0000000
--- a/examples/consumer/retry/concurrent/main.go
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// use concurrent consumer model, when Subscribe function return consumer.ConsumeRetryLater, the message will be
-// send to RocketMQ retry topic. we could set DelayLevelWhenNextConsume in ConsumeConcurrentlyContext, which used to
-// indicate the delay of message re-send to origin topic from retry topic.
-//
-// in this example, we always set DelayLevelWhenNextConsume=1, means that the message will be sent to origin topic after
-// 1s. in case of the unlimited retry, we will return consumer.ConsumeSuccess after ReconsumeTimes > 5
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- )
-
- // The DelayLevel specify the waiting time that before next reconsume,
- // and it range is from 1 to 18 now.
- //
- // The time of each level is the value of indexing of {level-1} in [1s, 5s, 10s, 30s,
- // 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 1h, 2h]
- delayLevel := 1
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback len: %d \n", len(msgs))
-
- concurrentCtx, _ := primitive.GetConcurrentlyCtx(ctx)
- concurrentCtx.DelayLevelWhenNextConsume = delayLevel // only run when return consumer.ConsumeRetryLater
-
- for _, msg := range msgs {
- if msg.ReconsumeTimes > 5 {
- fmt.Printf("msg ReconsumeTimes > 5. msg: %v", msg)
- return consumer.ConsumeSuccess, nil
- } else {
- fmt.Printf("subscribe callback: %v \n", msg)
- }
- }
- return consumer.ConsumeRetryLater, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shundown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/retry/order/main.go b/examples/consumer/retry/order/main.go
deleted file mode 100644
index e2d13bf..0000000
--- a/examples/consumer/retry/order/main.go
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * use orderly consumer model, when Subscribe function return consumer.SuspendCurrentQueueAMoment, it will be re-send to
- * local msg queue for later consume if msg.ReconsumeTimes < MaxReconsumeTimes, otherwise, it will be send to rocketmq
- * DLQ topic, we should manually resolve the msg.
- */
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerOrder(true),
- consumer.WithMaxReconsumeTimes(5),
- )
-
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
- fmt.Printf("orderly context: %v\n", orderlyCtx)
- fmt.Printf("subscribe orderly callback len: %d \n", len(msgs))
-
- for _, msg := range msgs {
- if msg.ReconsumeTimes > 5 {
- fmt.Printf("msg ReconsumeTimes > 5. msg: %v", msg)
- } else {
- fmt.Printf("subscribe orderly callback: %v \n", msg)
- }
- }
- return consumer.SuspendCurrentQueueAMoment, nil
-
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shundown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/simple/main.go b/examples/consumer/simple/main.go
deleted file mode 100644
index 70f35b3..0000000
--- a/examples/consumer/simple/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- for i := range msgs {
- fmt.Printf("subscribe callback: %v \n", msgs[i])
- }
-
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/strategy/main.go b/examples/consumer/strategy/main.go
deleted file mode 100644
index 269ce11..0000000
--- a/examples/consumer/strategy/main.go
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithStrategy(consumer.AllocateByAveragely),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/tag/main.go b/examples/consumer/tag/main.go
deleted file mode 100644
index ec16f51..0000000
--- a/examples/consumer/tag/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- selector := consumer.MessageSelector{
- Type: consumer.TAG,
- Expression: "TagA || TagC",
- }
- err := c.Subscribe("TopicTest", selector, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/trace/main.go b/examples/consumer/trace/main.go
deleted file mode 100644
index 97f22ed..0000000
--- a/examples/consumer/trace/main.go
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- namesrvs := []string{"127.0.0.1:9876"}
- traceCfg := &primitive.TraceConfig{
- Access: primitive.Local,
- Resolver: primitive.NewPassthroughResolver(namesrvs),
- }
-
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithTrace(traceCfg),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
-
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/acl/main.go b/examples/producer/acl/main.go
deleted file mode 100644
index 96881d3..0000000
--- a/examples/producer/acl/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, err := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- )
-
- if err != nil {
- fmt.Println("init producer error: " + err.Error())
- os.Exit(0)
- }
-
- err = p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 100000; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/async/main.go b/examples/producer/async/main.go
deleted file mode 100644
index 2a5182c..0000000
--- a/examples/producer/async/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "sync"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-// Package main implements a async producer to send message.
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithQueueSelector(producer.NewManualQueueSelector()))
-
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- var wg sync.WaitGroup
- for i := 0; i < 10; i++ {
- wg.Add(1)
- err := p.SendAsync(context.Background(),
- func(ctx context.Context, result *primitive.SendResult, e error) {
- if e != nil {
- fmt.Printf("receive message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", result.String())
- }
- wg.Done()
- }, primitive.NewMessage("test", []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- }
- }
- wg.Wait()
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/batch/main.go b/examples/producer/batch/main.go
deleted file mode 100644
index dc591a1..0000000
--- a/examples/producer/batch/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- var msgs []*primitive.Message
- for i := 0; i < 10; i++ {
- msgs = append(msgs, primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client! num: "+strconv.Itoa(i))))
- }
-
- res, err := p.SendSync(context.Background(), msgs...)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/delay/main.go b/examples/producer/delay/main.go
deleted file mode 100644
index aadbb70..0000000
--- a/examples/producer/delay/main.go
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 10; i++ {
- msg := primitive.NewMessage("test", []byte("Hello RocketMQ Go Client!"))
- msg.WithDelayTimeLevel(3)
- res, err := p.SendSync(context.Background(), msg)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/interceptor/main.go b/examples/producer/interceptor/main.go
deleted file mode 100644
index c40f9a0..0000000
--- a/examples/producer/interceptor/main.go
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithInterceptor(UserFirstInterceptor(), UserSecondInterceptor()),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 10; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
-
-func UserFirstInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- fmt.Printf("user first interceptor before invoke: req:%v\n", req)
- err := next(ctx, req, reply)
- fmt.Printf("user first interceptor after invoke: req: %v, reply: %v \n", req, reply)
- return err
- }
-}
-
-func UserSecondInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- fmt.Printf("user second interceptor before invoke: req: %v\n", req)
- err := next(ctx, req, reply)
- fmt.Printf("user second interceptor after invoke: req: %v, reply: %v \n", req, reply)
- return err
- }
-}
diff --git a/examples/producer/namespace/main.go b/examples/producer/namespace/main.go
deleted file mode 100644
index 9124284..0000000
--- a/examples/producer/namespace/main.go
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, err := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- producer.WithNamespace("namespace"),
- )
-
- if err != nil {
- fmt.Println("init producer error: " + err.Error())
- os.Exit(0)
- }
-
- err = p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 100000; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/simple/main.go b/examples/producer/simple/main.go
deleted file mode 100644
index 6fd3364..0000000
--- a/examples/producer/simple/main.go
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-// Package main implements a simple producer to send message.
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- topic := "test"
-
- for i := 0; i < 10; i++ {
- msg := &primitive.Message{
- Topic: topic,
- Body: []byte("Hello RocketMQ Go Client! " + strconv.Itoa(i)),
- }
- res, err := p.SendSync(context.Background(), msg)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/tag/main.go b/examples/producer/tag/main.go
deleted file mode 100644
index 7ce8559..0000000
--- a/examples/producer/tag/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- tags := []string{"TagA", "TagB", "TagC"}
- for i := 0; i < 3; i++ {
- tag := tags[i%3]
- msg := primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!"))
- msg.WithTag(tag)
-
- res, err := p.SendSync(context.Background(), msg)
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/trace/main.go b/examples/producer/trace/main.go
deleted file mode 100644
index b741704..0000000
--- a/examples/producer/trace/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- namesrvs := []string{"127.0.0.1:9876"}
- traceCfg := &primitive.TraceConfig{
- Access: primitive.Local,
- Resolver: primitive.NewPassthroughResolver(namesrvs),
- }
-
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithTrace(traceCfg))
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 1; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
-
- time.Sleep(10 * time.Second)
-
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/transaction/main.go b/examples/producer/transaction/main.go
deleted file mode 100644
index 05b6c52..0000000
--- a/examples/producer/transaction/main.go
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-type DemoListener struct {
- localTrans *sync.Map
- transactionIndex int32
-}
-
-func NewDemoListener() *DemoListener {
- return &DemoListener{
- localTrans: new(sync.Map),
- }
-}
-
-func (dl *DemoListener) ExecuteLocalTransaction(msg *primitive.Message) primitive.LocalTransactionState {
- nextIndex := atomic.AddInt32(&dl.transactionIndex, 1)
- fmt.Printf("nextIndex: %v for transactionID: %v\n", nextIndex, msg.TransactionId)
- status := nextIndex % 3
- dl.localTrans.Store(msg.TransactionId, primitive.LocalTransactionState(status+1))
-
- fmt.Printf("dl")
- return primitive.UnknowState
-}
-
-func (dl *DemoListener) CheckLocalTransaction(msg *primitive.MessageExt) primitive.LocalTransactionState {
- fmt.Printf("%v msg transactionID : %v\n", time.Now(), msg.TransactionId)
- v, existed := dl.localTrans.Load(msg.TransactionId)
- if !existed {
- fmt.Printf("unknow msg: %v, return Commit", msg)
- return primitive.CommitMessageState
- }
- state := v.(primitive.LocalTransactionState)
- switch state {
- case 1:
- fmt.Printf("checkLocalTransaction COMMIT_MESSAGE: %v\n", msg)
- return primitive.CommitMessageState
- case 2:
- fmt.Printf("checkLocalTransaction ROLLBACK_MESSAGE: %v\n", msg)
- return primitive.RollbackMessageState
- case 3:
- fmt.Printf("checkLocalTransaction unknow: %v\n", msg)
- return primitive.UnknowState
- default:
- fmt.Printf("checkLocalTransaction default COMMIT_MESSAGE: %v\n", msg)
- return primitive.CommitMessageState
- }
-}
-
-func main() {
- p, _ := rocketmq.NewTransactionProducer(
- NewDemoListener(),
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(1),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s\n", err.Error())
- os.Exit(1)
- }
-
- for i := 0; i < 10; i++ {
- res, err := p.SendMessageInTransaction(context.Background(),
- primitive.NewMessage("TopicTest5", []byte("Hello RocketMQ again "+strconv.Itoa(i))))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- time.Sleep(5 * time.Minute)
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/go.mod b/go.mod
deleted file mode 100644
index 0aac6ba..0000000
--- a/go.mod
+++ /dev/null
@@ -1,17 +0,0 @@
-module github.com/apache/rocketmq-client-go/v2
-
-go 1.13
-
-require (
- github.com/emirpasic/gods v1.12.0
- github.com/golang/mock v1.3.1
- github.com/json-iterator/go v1.1.9
- github.com/pkg/errors v0.8.1
- github.com/sirupsen/logrus v1.4.1
- github.com/smartystreets/goconvey v1.6.4
- github.com/stathat/consistent v1.0.0
- github.com/stretchr/testify v1.3.0
- github.com/tidwall/gjson v1.13.0
- go.uber.org/atomic v1.5.1
- stathat.com/c/consistent v1.0.0 // indirect
-)
diff --git a/go.sum b/go.sum
deleted file mode 100644
index 157f415..0000000
--- a/go.sum
+++ /dev/null
@@ -1,63 +0,0 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
-github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
-github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
-github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
-go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
-stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
diff --git a/internal/callback.go b/internal/callback.go
deleted file mode 100644
index fea11b4..0000000
--- a/internal/callback.go
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "net"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// remotingClient callback TransactionProducer
-type CheckTransactionStateCallback struct {
- Addr net.Addr
- Msg *primitive.MessageExt
- Header CheckTransactionStateRequestHeader
-}
diff --git a/internal/client.go b/internal/client.go
deleted file mode 100644
index c7f3e58..0000000
--- a/internal/client.go
+++ /dev/null
@@ -1,893 +0,0 @@
... 11153 lines suppressed ...