You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by di...@apache.org on 2022/04/11 02:50:46 UTC
[rocketmq-client-go] 01/01: init new empty branch feature.
This is an automated email from the ASF dual-hosted git repository.
dinglei pushed a commit to branch feature
in repository https://gitbox.apache.org/repos/asf/rocketmq-client-go.git
commit 4999236ed73d5972fd23b1f766503aa1d9608717
Author: ShannonDing <li...@163.com>
AuthorDate: Mon Apr 11 10:50:15 2022 +0800
init new empty branch feature.
---
admin/admin.go | 210 -----
admin/option.go | 131 ---
api.go | 137 ---
benchmark/consumer.go | 277 ------
benchmark/main.go | 76 --
benchmark/message.go | 34 -
benchmark/producer.go | 290 -------
benchmark/stable.go | 280 -------
config.go | 17 -
consumer/consumer.go | 1063 -----------------------
consumer/consumer_test.go | 171 ----
consumer/interceptor.go | 115 ---
consumer/lock.go | 37 -
consumer/mock_offset_store.go | 94 ---
consumer/offset_store.go | 402 ---------
consumer/offset_store_test.go | 243 ------
consumer/option.go | 276 ------
consumer/process_queue.go | 397 ---------
consumer/pull_consumer.go | 256 ------
consumer/pull_consumer_test.go | 18 -
consumer/push_consumer.go | 1254 ----------------------------
consumer/push_consumer_test.go | 96 ---
consumer/statistics.go | 485 -----------
consumer/statistics_test.go | 214 -----
consumer/strategy.go | 247 ------
consumer/strategy_test.go | 487 -----------
docs/Introduction.md | 130 ---
docs/client-design.gliffy | 1 -
docs/feature.md | 86 --
docs/images/client-design.png | Bin 30513 -> 0 bytes
docs/zh/native-design_zh.md | 114 ---
docs/zh/rocketmq-protocol_zh.md | 117 ---
errors/errors.go | 51 --
examples/admin/topic/main.go | 64 --
examples/consumer/acl/main.go | 64 --
examples/consumer/broadcast/main.go | 57 --
examples/consumer/delay/main.go | 60 --
examples/consumer/interceptor/main.go | 83 --
examples/consumer/namespace/main.go | 65 --
examples/consumer/orderly/main.go | 60 --
examples/consumer/pull/main.go | 72 --
examples/consumer/retry/concurrent/main.go | 81 --
examples/consumer/retry/order/main.go | 76 --
examples/consumer/simple/main.go | 58 --
examples/consumer/strategy/main.go | 56 --
examples/consumer/tag/main.go | 58 --
examples/consumer/trace/main.go | 63 --
examples/producer/acl/main.go | 65 --
examples/producer/async/main.go | 65 --
examples/producer/batch/main.go | 58 --
examples/producer/delay/main.go | 55 --
examples/producer/interceptor/main.go | 74 --
examples/producer/namespace/main.go | 66 --
examples/producer/simple/main.go | 61 --
examples/producer/tag/main.go | 58 --
examples/producer/trace/main.go | 64 --
examples/producer/transaction/main.go | 106 ---
go.mod | 17 -
go.sum | 63 --
internal/callback.go | 31 -
internal/client.go | 893 --------------------
internal/constants.go | 29 -
internal/mock_client.go | 430 ----------
internal/mock_namesrv.go | 193 -----
internal/model.go | 406 ---------
internal/model_test.go | 484 -----------
internal/mq_version.go | 22 -
internal/namesrv.go | 185 ----
internal/namesrv_test.go | 149 ----
internal/perm.go | 58 --
internal/remote/codec.go | 521 ------------
internal/remote/codec_test.go | 378 ---------
internal/remote/future.go | 68 --
internal/remote/interceptor.go | 83 --
internal/remote/interceptor_test.go | 29 -
internal/remote/mock_remote_client.go | 123 ---
internal/remote/remote_client.go | 316 -------
internal/remote/remote_client_test.go | 372 ---------
internal/remote/rpchook.go | 23 -
internal/remote/tcp_conn.go | 61 --
internal/request.go | 480 -----------
internal/response.go | 49 --
internal/route.go | 668 ---------------
internal/route_test.go | 137 ---
internal/trace.go | 526 ------------
internal/trace_test.go | 125 ---
internal/transaction.go | 21 -
internal/utils/compression.go | 86 --
internal/utils/compression_test.go | 97 ---
internal/utils/errors.go | 30 -
internal/utils/files.go | 89 --
internal/utils/math.go | 32 -
internal/utils/net.go | 70 --
internal/utils/net_test.go | 24 -
internal/utils/set.go | 103 ---
internal/utils/string.go | 30 -
internal/validators.go | 43 -
primitive/auth.go | 28 -
primitive/base.go | 129 ---
primitive/base_test.go | 76 --
primitive/ctx.go | 173 ----
primitive/errors.go | 61 --
primitive/interceptor.go | 51 --
primitive/message.go | 569 -------------
primitive/message_test.go | 38 -
primitive/nsresolver.go | 223 -----
primitive/nsresolver_test.go | 136 ---
primitive/pool.go | 57 --
primitive/result.go | 128 ---
primitive/result_test.go | 74 --
primitive/trace.go | 28 -
producer/interceptor.go | 103 ---
producer/option.go | 164 ----
producer/producer.go | 602 -------------
producer/producer_test.go | 358 --------
producer/selector.go | 122 ---
producer/selector_test.go | 78 --
rlog/log.go | 180 ----
118 files changed, 20317 deletions(-)
diff --git a/admin/admin.go b/admin/admin.go
deleted file mode 100644
index 06908f4..0000000
--- a/admin/admin.go
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package admin
-
-import (
- "context"
- "fmt"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type Admin interface {
- CreateTopic(ctx context.Context, opts ...OptionCreate) error
- DeleteTopic(ctx context.Context, opts ...OptionDelete) error
- //TODO
- //TopicList(ctx context.Context, mq *primitive.MessageQueue) (*remote.RemotingCommand, error)
- //GetBrokerClusterInfo(ctx context.Context) (*remote.RemotingCommand, error)
- Close() error
-}
-
-// TODO: move outdated context to ctx
-type adminOptions struct {
- internal.ClientOptions
-}
-
-type AdminOption func(options *adminOptions)
-
-func defaultAdminOptions() *adminOptions {
- opts := &adminOptions{
- ClientOptions: internal.DefaultClientOptions(),
- }
- opts.GroupName = "TOOLS_ADMIN"
- opts.InstanceName = time.Now().String()
- return opts
-}
-
-// WithResolver nameserver resolver to fetch nameserver addr
-func WithResolver(resolver primitive.NsResolver) AdminOption {
- return func(options *adminOptions) {
- options.Resolver = resolver
- }
-}
-
-type admin struct {
- cli internal.RMQClient
-
- opts *adminOptions
-
- closeOnce sync.Once
-}
-
-// NewAdmin initialize admin
-func NewAdmin(opts ...AdminOption) (Admin, error) {
- defaultOpts := defaultAdminOptions()
- for _, opt := range opts {
- opt(defaultOpts)
- }
- namesrv, err := internal.NewNamesrv(defaultOpts.Resolver)
- defaultOpts.Namesrv = namesrv
- if err != nil {
- return nil, err
- }
-
- cli := internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil)
- if cli == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = cli.GetNameSrv()
- //log.Printf("Client: %#v", namesrv.srvs)
- return &admin{
- cli: cli,
- opts: defaultOpts,
- }, nil
-}
-
-// CreateTopic create topic.
-// TODO: another implementation like sarama, without brokerAddr as input
-func (a *admin) CreateTopic(ctx context.Context, opts ...OptionCreate) error {
- cfg := defaultTopicConfigCreate()
- for _, apply := range opts {
- apply(&cfg)
- }
-
- request := &internal.CreateTopicRequestHeader{
- Topic: cfg.Topic,
- DefaultTopic: cfg.DefaultTopic,
- ReadQueueNums: cfg.ReadQueueNums,
- WriteQueueNums: cfg.WriteQueueNums,
- Perm: cfg.Perm,
- TopicFilterType: cfg.TopicFilterType,
- TopicSysFlag: cfg.TopicSysFlag,
- Order: cfg.Order,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqCreateTopic, request, nil)
- _, err := a.cli.InvokeSync(ctx, cfg.BrokerAddr, cmd, 5*time.Second)
- if err != nil {
- rlog.Error("create topic error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- } else {
- rlog.Info("create topic success", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- })
- }
- return err
-}
-
-// DeleteTopicInBroker delete topic in broker.
-func (a *admin) deleteTopicInBroker(ctx context.Context, topic string, brokerAddr string) (*remote.RemotingCommand, error) {
- request := &internal.DeleteTopicRequestHeader{
- Topic: topic,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqDeleteTopicInBroker, request, nil)
- return a.cli.InvokeSync(ctx, brokerAddr, cmd, 5*time.Second)
-}
-
-// DeleteTopicInNameServer delete topic in nameserver.
-func (a *admin) deleteTopicInNameServer(ctx context.Context, topic string, nameSrvAddr string) (*remote.RemotingCommand, error) {
- request := &internal.DeleteTopicRequestHeader{
- Topic: topic,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqDeleteTopicInNameSrv, request, nil)
- return a.cli.InvokeSync(ctx, nameSrvAddr, cmd, 5*time.Second)
-}
-
-// DeleteTopic delete topic in both broker and nameserver.
-func (a *admin) DeleteTopic(ctx context.Context, opts ...OptionDelete) error {
- cfg := defaultTopicConfigDelete()
- for _, apply := range opts {
- apply(&cfg)
- }
- //delete topic in broker
- if cfg.BrokerAddr == "" {
- a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- cfg.BrokerAddr = a.cli.GetNameSrv().FindBrokerAddrByTopic(cfg.Topic)
- }
-
- if _, err := a.deleteTopicInBroker(ctx, cfg.Topic, cfg.BrokerAddr); err != nil {
- rlog.Error("delete topic in broker error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- return err
- }
-
- //delete topic in nameserver
- if len(cfg.NameSrvAddr) == 0 {
- a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- cfg.NameSrvAddr = a.cli.GetNameSrv().AddrList()
- _, _, err := a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- if err != nil {
- rlog.Error("delete topic in nameserver error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyUnderlayError: err,
- })
- }
- cfg.NameSrvAddr = a.cli.GetNameSrv().AddrList()
- }
-
- for _, nameSrvAddr := range cfg.NameSrvAddr {
- if _, err := a.deleteTopicInNameServer(ctx, cfg.Topic, nameSrvAddr); err != nil {
- rlog.Error("delete topic in nameserver error", map[string]interface{}{
- "nameServer": nameSrvAddr,
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyUnderlayError: err,
- })
- return err
- }
- }
- rlog.Info("delete topic success", map[string]interface{}{
- "nameServer": cfg.NameSrvAddr,
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- })
- return nil
-}
-
-func (a *admin) Close() error {
- a.closeOnce.Do(func() {
- a.cli.Shutdown()
- })
- return nil
-}
diff --git a/admin/option.go b/admin/option.go
deleted file mode 100644
index d5a648e..0000000
--- a/admin/option.go
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package admin
-
-func defaultTopicConfigCreate() TopicConfigCreate {
- opts := TopicConfigCreate{
- DefaultTopic: "defaultTopic",
- ReadQueueNums: 8,
- WriteQueueNums: 8,
- Perm: 6,
- TopicFilterType: "SINGLE_TAG",
- TopicSysFlag: 0,
- Order: false,
- }
- return opts
-}
-
-type TopicConfigCreate struct {
- Topic string
- BrokerAddr string
- DefaultTopic string
- ReadQueueNums int
- WriteQueueNums int
- Perm int
- TopicFilterType string
- TopicSysFlag int
- Order bool
-}
-
-type OptionCreate func(*TopicConfigCreate)
-
-func WithTopicCreate(Topic string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Topic = Topic
- }
-}
-
-func WithBrokerAddrCreate(BrokerAddr string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.BrokerAddr = BrokerAddr
- }
-}
-
-func WithReadQueueNums(ReadQueueNums int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.ReadQueueNums = ReadQueueNums
- }
-}
-
-func WithWriteQueueNums(WriteQueueNums int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.WriteQueueNums = WriteQueueNums
- }
-}
-
-func WithPerm(Perm int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Perm = Perm
- }
-}
-
-func WithTopicFilterType(TopicFilterType string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.TopicFilterType = TopicFilterType
- }
-}
-
-func WithTopicSysFlag(TopicSysFlag int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.TopicSysFlag = TopicSysFlag
- }
-}
-
-func WithOrder(Order bool) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Order = Order
- }
-}
-
-func defaultTopicConfigDelete() TopicConfigDelete {
- opts := TopicConfigDelete{}
- return opts
-}
-
-type TopicConfigDelete struct {
- Topic string
- ClusterName string
- NameSrvAddr []string
- BrokerAddr string
-}
-
-type OptionDelete func(*TopicConfigDelete)
-
-func WithTopicDelete(Topic string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.Topic = Topic
- }
-}
-
-func WithBrokerAddrDelete(BrokerAddr string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.BrokerAddr = BrokerAddr
- }
-}
-
-func WithClusterName(ClusterName string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.ClusterName = ClusterName
- }
-}
-
-func WithNameSrvAddr(NameSrvAddr []string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.NameSrvAddr = NameSrvAddr
- }
-}
diff --git a/api.go b/api.go
deleted file mode 100644
index 31f58d5..0000000
--- a/api.go
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package rocketmq
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/errors"
-
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-type Producer interface {
- Start() error
- Shutdown() error
- SendSync(ctx context.Context, mq ...*primitive.Message) (*primitive.SendResult, error)
- SendAsync(ctx context.Context, mq func(ctx context.Context, result *primitive.SendResult, err error),
- msg ...*primitive.Message) error
- SendOneWay(ctx context.Context, mq ...*primitive.Message) error
-}
-
-func NewProducer(opts ...producer.Option) (Producer, error) {
- return producer.NewDefaultProducer(opts...)
-}
-
-type TransactionProducer interface {
- Start() error
- Shutdown() error
- SendMessageInTransaction(ctx context.Context, mq *primitive.Message) (*primitive.TransactionSendResult, error)
-}
-
-func NewTransactionProducer(listener primitive.TransactionListener, opts ...producer.Option) (TransactionProducer, error) {
- return producer.NewTransactionProducer(listener, opts...)
-}
-
-type PushConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be sync to broker before process exit
- Shutdown() error
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (consumer.ConsumeResult, error)) error
-
- // Unsubscribe a topic
- Unsubscribe(topic string) error
-}
-
-func NewPushConsumer(opts ...consumer.Option) (PushConsumer, error) {
- return consumer.NewPushConsumer(opts...)
-}
-
-type PullConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be commit to broker before process exit
- Shutdown() error
-
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector) error
-
- // Unsubscribe a topic
- Unsubscribe(topic string) error
-
- // MessageQueues get MessageQueue list about for a given topic. This method will issue a remote call to the server
- // if it does not already have any MessageQueue about the given topic.
- MessageQueues(topic string) []primitive.MessageQueue
-
- // Pull message for the topic specified. It is an error to not have subscribed to any topics before pull for message
- //
- // Specified numbers of messages is returned if message greater that numbers, and the offset will auto forward.
- // It means that if you meeting messages consuming failed, you should process failed messages by yourself.
- Pull(ctx context.Context, topic string, numbers int) (*primitive.PullResult, error)
-
- // Pull message for the topic specified from a specified MessageQueue and offset. It is an error to not have
- // subscribed to any topics before pull for message. the method will not affect the offset recorded
- //
- // Specified numbers of messages is returned.
- PullFrom(ctx context.Context, mq primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error)
-
- // Lookup offset for the given message queue by timestamp. The returned offset for the message queue is the
- // earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message
- // queue.
- //
- // Timestamp must be millisecond level, if you want to lookup the earliest offset of the mq, you could set the
- // timestamp 0, and if you want to the latest offset the mq, you could set the timestamp math.MaxInt64.
- Lookup(ctx context.Context, mq primitive.MessageQueue, timestamp int64) (int64, error)
-
- // Commit the offset of specified mqs to broker, if auto-commit is disable, you must commit the offset manually.
- Commit(ctx context.Context, mqs ...primitive.MessageQueue) (int64, error)
-
- // CommittedOffset return the offset of specified Message
- CommittedOffset(mq primitive.MessageQueue) (int64, error)
-
- // Seek set offset of the mq, if you wanna re-consuming your message form one position, the method may help you.
- // if you want re-consuming from one time, you cloud Lookup() then seek it.
- Seek(mq primitive.MessageQueue, offset int64) error
-
- // Pause consuming for specified MessageQueues, after pause, client will not fetch any message from the specified
- // message queues
- //
- // Note that this method does not affect message queue subscription. In particular, it does not cause a group
- // rebalance.
- //
- // if a MessageQueue belong a topic that has not been subscribed, an error will be returned
- //Pause(mqs ...primitive.MessageQueue) error
-
- // Resume specified message queues which have been paused with Pause, if a MessageQueue that not paused,
- // it will be ignored. if not subscribed, an error will be returned
- //Resume(mqs ...primitive.MessageQueue) error
-}
-
-// The PullConsumer has not implemented completely, if you want have an experience of PullConsumer, you could use
-// consumer.NewPullConsumer(...), but it may changed in the future.
-//
-// The PullConsumer will be supported in next release
-func NewPullConsumer(opts ...consumer.Option) (PullConsumer, error) {
- return nil, errors.ErrPullConsumer
-}
diff --git a/benchmark/consumer.go b/benchmark/consumer.go
deleted file mode 100644
index 907a1e7..0000000
--- a/benchmark/consumer.go
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "context"
- "flag"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "sync"
- "sync/atomic"
- "syscall"
- "time"
-)
-
-type statiBenchmarkConsumerSnapshot struct {
- receiveMessageTotal int64
- born2ConsumerTotalRT int64
- store2ConsumerTotalRT int64
- born2ConsumerMaxRT int64
- store2ConsumerMaxRT int64
- createdAt time.Time
- next *statiBenchmarkConsumerSnapshot
-}
-
-type consumeSnapshots struct {
- sync.RWMutex
- head, tail, cur *statiBenchmarkConsumerSnapshot
- len int
-}
-
-func (s *consumeSnapshots) takeSnapshot() {
- b := s.cur
- sn := new(statiBenchmarkConsumerSnapshot)
- sn.receiveMessageTotal = atomic.LoadInt64(&b.receiveMessageTotal)
- sn.born2ConsumerMaxRT = atomic.LoadInt64(&b.born2ConsumerMaxRT)
- sn.born2ConsumerTotalRT = atomic.LoadInt64(&b.born2ConsumerTotalRT)
- sn.store2ConsumerMaxRT = atomic.LoadInt64(&b.store2ConsumerMaxRT)
- sn.store2ConsumerTotalRT = atomic.LoadInt64(&b.store2ConsumerTotalRT)
- sn.createdAt = time.Now()
-
- s.Lock()
- if s.tail != nil {
- s.tail.next = sn
- }
- s.tail = sn
- if s.head == nil {
- s.head = s.tail
- }
-
- s.len++
- if s.len > 10 {
- s.head = s.head.next
- s.len--
- }
- s.Unlock()
-}
-
-func (s *consumeSnapshots) printStati() {
- s.RLock()
- if s.len < 10 {
- s.RUnlock()
- return
- }
-
- f, l := s.head, s.tail
- respSucCount := float64(l.receiveMessageTotal - f.receiveMessageTotal)
- consumeTps := respSucCount / l.createdAt.Sub(f.createdAt).Seconds()
- avgB2CRT := float64(l.born2ConsumerTotalRT-f.born2ConsumerTotalRT) / respSucCount
- avgS2CRT := float64(l.store2ConsumerTotalRT-f.store2ConsumerTotalRT) / respSucCount
- s.RUnlock()
-
- rlog.Info("Benchmark Consumer Snapshot", map[string]interface{}{
- "consumeTPS": int64(consumeTps),
- "average(B2C)RT": avgB2CRT,
- "average(S2C)RT": avgS2CRT,
- "max(B2C)RT": l.born2ConsumerMaxRT,
- "max(S2C)RT": l.store2ConsumerMaxRT,
- })
-}
-
-type consumerBenchmark struct {
- topic string
- groupPrefix string
- nameSrv string
- isPrefixEnable bool
- filterType string
- expression string
- testMinutes int
- instanceCount int
-
- flags *flag.FlagSet
-
- groupID string
-}
-
-func init() {
- c := &consumerBenchmark{}
- flags := flag.NewFlagSet("consumer", flag.ExitOnError)
- c.flags = flags
-
- flags.StringVar(&c.topic, "t", "BenchmarkTest", "topic")
- flags.StringVar(&c.groupPrefix, "g", "benchmark_consumer", "group prefix")
- flags.StringVar(&c.nameSrv, "n", "", "namesrv address list, separated by comma")
- flags.BoolVar(&c.isPrefixEnable, "p", true, "group prefix is enable")
- flags.StringVar(&c.filterType, "f", "", "filter type,options:TAG|SQL92, or empty")
- flags.StringVar(&c.expression, "e", "*", "expression")
- flags.IntVar(&c.testMinutes, "m", 10, "test minutes")
- flags.IntVar(&c.instanceCount, "i", 1, "instance count")
-
- registerCommand("consumer", c)
-}
-
-func (bc *consumerBenchmark) consumeMsg(stati *statiBenchmarkConsumerSnapshot, exit chan struct{}) {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName(bc.groupID),
- consumer.WithNameServer([]string{bc.nameSrv}),
- )
- if err != nil {
- panic("new push consumer error:" + err.Error())
- }
-
- selector := consumer.MessageSelector{}
- err = c.Subscribe(bc.topic, selector, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- for _, msg := range msgs {
- atomic.AddInt64(&stati.receiveMessageTotal, 1)
- now := time.Now().UnixNano() / int64(time.Millisecond)
- b2cRT := now - msg.BornTimestamp
- atomic.AddInt64(&stati.born2ConsumerTotalRT, b2cRT)
- s2cRT := now - msg.StoreTimestamp
- atomic.AddInt64(&stati.store2ConsumerTotalRT, s2cRT)
-
- for {
- old := atomic.LoadInt64(&stati.born2ConsumerMaxRT)
- if old >= b2cRT || atomic.CompareAndSwapInt64(&stati.born2ConsumerMaxRT, old, b2cRT) {
- break
- }
- }
-
- for {
- old := atomic.LoadInt64(&stati.store2ConsumerMaxRT)
- if old >= s2cRT || atomic.CompareAndSwapInt64(&stati.store2ConsumerMaxRT, old, s2cRT) {
- break
- }
- }
- }
- return consumer.ConsumeSuccess, nil
- })
-
- rlog.Info("Test Start", nil)
- c.Start()
- select {
- case <-exit:
- c.Shutdown()
- return
- }
-}
-
-func (bc *consumerBenchmark) run(args []string) {
- bc.flags.Parse(args)
- if bc.topic == "" {
- rlog.Error("Empty Topic", nil)
- bc.usage()
- return
- }
-
- if bc.groupPrefix == "" {
- rlog.Error("Empty Group Prefix", nil)
- bc.usage()
- return
- }
-
- if bc.nameSrv == "" {
- rlog.Error("Empty Nameserver", nil)
- bc.usage()
- return
- }
-
- if bc.testMinutes <= 0 {
- rlog.Error("Test Time Must Be Positive Integer", nil)
- bc.usage()
- return
- }
-
- if bc.instanceCount <= 0 {
- rlog.Error("Thread Count Must Be Positive Integer", nil)
- bc.usage()
- return
- }
-
- bc.groupID = bc.groupPrefix
- if bc.isPrefixEnable {
- bc.groupID += fmt.Sprintf("_%d", time.Now().UnixNano()/int64(time.Millisecond)%100)
- }
-
- stati := statiBenchmarkConsumerSnapshot{}
- snapshots := consumeSnapshots{cur: &stati}
- exitChan := make(chan struct{})
-
- wg := sync.WaitGroup{}
-
- wg.Add(1)
- go func() {
- bc.consumeMsg(&stati, exitChan)
- wg.Done()
- }()
-
- // snapshot
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second)
- for {
- select {
- case <-ticker.C:
- snapshots.takeSnapshot()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- // print statistic
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second * 10)
- for {
- select {
- case <-ticker.C:
- snapshots.printStati()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- select {
- case <-time.Tick(time.Minute * time.Duration(bc.testMinutes)):
- case <-signalChan:
- }
-
- close(exitChan)
- wg.Wait()
- snapshots.takeSnapshot()
- snapshots.printStati()
- rlog.Info("Test Done", nil)
-}
-
-func (bc *consumerBenchmark) usage() {
- bc.flags.Usage()
-}
diff --git a/benchmark/main.go b/benchmark/main.go
deleted file mode 100644
index 79eca8f..0000000
--- a/benchmark/main.go
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
-)
-
-type command interface {
- usage()
- run(args []string)
-}
-
-var (
- cmds = map[string]command{}
-)
-
-func registerCommand(name string, cmd command) {
- if cmd == nil {
- panic("empty command")
- }
-
- _, ok := cmds[name]
- if ok {
- panic(fmt.Sprintf("%s command existed", name))
- }
-
- cmds[name] = cmd
-}
-
-func usage() {
- rlog.Info("Command", map[string]interface{}{
- "name": os.Args[0],
- })
- for _, cmd := range cmds {
- cmd.usage()
- }
-}
-
-// go run *.go [command name] [command args]
-func main() {
- if len(os.Args) < 2 {
- rlog.Error("Lack Command Name", nil)
- usage()
- return
- }
-
- name := os.Args[1]
- cmd, ok := cmds[name]
- if !ok {
- rlog.Error("Command Isn't Supported", map[string]interface{}{
- "command": name,
- })
- usage()
- return
- }
-
- cmd.run(os.Args[2:])
-}
diff --git a/benchmark/message.go b/benchmark/message.go
deleted file mode 100644
index d5690fe..0000000
--- a/benchmark/message.go
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "strings"
-
-var (
- longText = ""
- longTextLen = 0
-)
-
-func init() {
- longText = strings.Repeat("0123456789", 100)
- longTextLen = len(longText)
-}
-
-func buildMsg(size int) string {
- return longText[:size]
-}
diff --git a/benchmark/producer.go b/benchmark/producer.go
deleted file mode 100644
index 7516352..0000000
--- a/benchmark/producer.go
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "context"
- "flag"
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "sync"
- "sync/atomic"
- "syscall"
- "time"
-)
-
-type statiBenchmarkProducerSnapshot struct {
- sendRequestSuccessCount int64
- sendRequestFailedCount int64
- receiveResponseSuccessCount int64
- receiveResponseFailedCount int64
- sendMessageSuccessTimeTotal int64
- sendMessageMaxRT int64
- createdAt time.Time
- next *statiBenchmarkProducerSnapshot
-}
-
-type produceSnapshots struct {
- sync.RWMutex
- head, tail, cur *statiBenchmarkProducerSnapshot
- len int
-}
-
-func (s *produceSnapshots) takeSnapshot() {
- b := s.cur
- sn := new(statiBenchmarkProducerSnapshot)
- sn.sendRequestSuccessCount = atomic.LoadInt64(&b.sendRequestSuccessCount)
- sn.sendRequestFailedCount = atomic.LoadInt64(&b.sendRequestFailedCount)
- sn.receiveResponseSuccessCount = atomic.LoadInt64(&b.receiveResponseSuccessCount)
- sn.receiveResponseFailedCount = atomic.LoadInt64(&b.receiveResponseFailedCount)
- sn.sendMessageSuccessTimeTotal = atomic.LoadInt64(&b.sendMessageSuccessTimeTotal)
- sn.sendMessageMaxRT = atomic.LoadInt64(&b.sendMessageMaxRT)
- sn.createdAt = time.Now()
-
- s.Lock()
- if s.tail != nil {
- s.tail.next = sn
- }
- s.tail = sn
- if s.head == nil {
- s.head = s.tail
- }
-
- s.len++
- if s.len > 10 {
- s.head = s.head.next
- s.len--
- }
- s.Unlock()
-}
-
-func (s *produceSnapshots) printStati() {
- s.RLock()
- if s.len < 10 {
- s.RUnlock()
- return
- }
-
- f, l := s.head, s.tail
- respSucCount := float64(l.receiveResponseSuccessCount - f.receiveResponseSuccessCount)
- sendTps := respSucCount / l.createdAt.Sub(f.createdAt).Seconds()
- avgRT := float64(l.sendMessageSuccessTimeTotal-f.sendMessageSuccessTimeTotal) / respSucCount
- maxRT := atomic.LoadInt64(&s.cur.sendMessageMaxRT)
- s.RUnlock()
-
- rlog.Info("Benchmark Producer Snapshot", map[string]interface{}{
- "sendTps": int64(sendTps),
- "maxRt": maxRT,
- "averageRt": avgRT,
- "sendFailed": l.sendRequestFailedCount,
- "responseFailed": l.receiveResponseFailedCount,
- "total": l.receiveResponseSuccessCount,
- })
-}
-
-type producerBenchmark struct {
- topic string
- nameSrv string
- groupID string
- instanceCount int
- testMinutes int
- bodySize int
-
- flags *flag.FlagSet
-}
-
-func init() {
- p := &producerBenchmark{}
- flags := flag.NewFlagSet("producer", flag.ExitOnError)
- p.flags = flags
-
- flags.StringVar(&p.topic, "t", "", "topic name")
- flags.StringVar(&p.nameSrv, "n", "", "nameserver address")
- flags.StringVar(&p.groupID, "g", "", "group id")
- flags.IntVar(&p.instanceCount, "i", 1, "instance count")
- flags.IntVar(&p.testMinutes, "m", 10, "test minutes")
- flags.IntVar(&p.bodySize, "s", 32, "body size")
-
- registerCommand("producer", p)
-}
-
-func (bp *producerBenchmark) produceMsg(stati *statiBenchmarkProducerSnapshot, exit chan struct{}) {
- p, err := rocketmq.NewProducer(
- producer.WithNameServer([]string{bp.nameSrv}),
- producer.WithRetry(2),
- )
-
- if err != nil {
- rlog.Error("New Producer Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return
- }
-
- err = p.Start()
-
- defer p.Shutdown()
-
- topic, tag := bp.topic, "benchmark-producer"
- msgStr := buildMsg(bp.bodySize)
-
-AGAIN:
- select {
- case <-exit:
- return
- default:
- }
-
- now := time.Now()
- r, err := p.SendSync(context.Background(), primitive.NewMessage(topic, []byte(msgStr)))
-
- if err != nil {
- rlog.Error("Send Message Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- goto AGAIN
- }
-
- if r.Status == primitive.SendOK {
- atomic.AddInt64(&stati.receiveResponseSuccessCount, 1)
- atomic.AddInt64(&stati.sendRequestSuccessCount, 1)
- currentRT := int64(time.Since(now) / time.Millisecond)
- atomic.AddInt64(&stati.sendMessageSuccessTimeTotal, currentRT)
- prevRT := atomic.LoadInt64(&stati.sendMessageMaxRT)
- for currentRT > prevRT {
- if atomic.CompareAndSwapInt64(&stati.sendMessageMaxRT, prevRT, currentRT) {
- break
- }
- prevRT = atomic.LoadInt64(&stati.sendMessageMaxRT)
- }
- goto AGAIN
- }
- rlog.Error("Send Message Error", map[string]interface{}{
- "topic": topic,
- "tag": tag,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- goto AGAIN
-}
-
-func (bp *producerBenchmark) run(args []string) {
- bp.flags.Parse(args)
-
- if bp.topic == "" {
- rlog.Error("Empty Topic", nil)
- bp.flags.Usage()
- return
- }
-
- if bp.groupID == "" {
- rlog.Error("Empty Group Id", nil)
- bp.flags.Usage()
- return
- }
-
- if bp.nameSrv == "" {
- rlog.Error("Empty Nameserver", nil)
- bp.flags.Usage()
- return
- }
- if bp.instanceCount <= 0 {
- rlog.Error("Instance Count Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
- if bp.testMinutes <= 0 {
- rlog.Error("Test Time Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
- if bp.bodySize <= 0 {
- rlog.Error("Body Size Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
-
- stati := statiBenchmarkProducerSnapshot{}
- snapshots := produceSnapshots{cur: &stati}
- exitChan := make(chan struct{})
- wg := sync.WaitGroup{}
-
- for i := 0; i < bp.instanceCount; i++ {
- i := i
- go func() {
- wg.Add(1)
- bp.produceMsg(&stati, exitChan)
- rlog.Info("Producer Done and Exit", map[string]interface{}{
- "id": i,
- })
- wg.Done()
- }()
- }
-
- // snapshot
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second)
- for {
- select {
- case <-ticker.C:
- snapshots.takeSnapshot()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- // print statistic
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second * 10)
- for {
- select {
- case <-ticker.C:
- snapshots.printStati()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- select {
- case <-time.Tick(time.Minute * time.Duration(bp.testMinutes)):
- case <-signalChan:
- }
-
- close(exitChan)
- wg.Wait()
- snapshots.takeSnapshot()
- snapshots.printStati()
- rlog.Info("Test Done", nil)
-}
-
-func (bp *producerBenchmark) usage() {
- bp.flags.Usage()
-}
diff --git a/benchmark/stable.go b/benchmark/stable.go
deleted file mode 100644
index 2659bc5..0000000
--- a/benchmark/stable.go
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "flag"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "syscall"
- "time"
-)
-
-type stableTest struct {
- nameSrv string
- topic string
- groupID string
- opIntervalSec int
- testMin int
-
- op func()
-
- flags *flag.FlagSet
-}
-
-func (st *stableTest) buildFlags(name string) {
- flags := flag.NewFlagSet(name, flag.ExitOnError)
- flags.StringVar(&st.topic, "t", "stable-test", "topic name")
- flags.StringVar(&st.nameSrv, "n", "", "nameserver address")
- flags.StringVar(&st.groupID, "g", "stable-test", "group id")
- flags.IntVar(&st.testMin, "m", 10, "test minutes")
- flags.IntVar(&st.opIntervalSec, "s", 1, "operation interval[produce/consume]")
-
- st.flags = flags
-}
-
-func (st *stableTest) checkFlag() error {
- if st.topic == "" {
- return errors.ErrEmptyTopic
- }
-
- if st.nameSrv == "" {
- return errors.ErrEmptyNameSrv
- }
-
- if st.groupID == "" {
- return errors.ErrEmptyGroupID
- }
-
- if st.testMin <= 0 {
- return errors.ErrTestMin
- }
-
- if st.opIntervalSec <= 0 {
- return errors.ErrOperationInterval
- }
-
- return nil
-}
-
-func (st *stableTest) run() {
- opTicker := time.NewTicker(time.Duration(st.opIntervalSec) * time.Second)
- closeChan := time.Tick(time.Duration(st.testMin) * time.Minute)
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- for {
- select {
- case <-signalChan:
- opTicker.Stop()
- rlog.Info("Test Done", nil)
- return
- case <-closeChan:
- opTicker.Stop()
- rlog.Info("Test Done", nil)
- return
- case <-opTicker.C:
- st.op()
- }
- }
-}
-
-type stableTestProducer struct {
- *stableTest
- bodySize int
-
- //p rocketmq.Producer
-}
-
-func (stp *stableTestProducer) buildFlags(name string) {
- stp.stableTest.buildFlags(name)
- stp.flags.IntVar(&stp.bodySize, "b", 32, "body size")
-}
-
-func (stp *stableTestProducer) checkFlag() error {
- err := stp.stableTest.checkFlag()
- if err != nil {
- return err
- }
- if stp.bodySize <= 0 {
- return errors.ErrMessageBody
- }
-
- return nil
-}
-
-func (stp *stableTestProducer) usage() {
- stp.flags.Usage()
-}
-
-func (stp *stableTestProducer) run(args []string) {
- err := stp.flags.Parse(args)
- if err != nil {
- rlog.Info("Parse Args Error", map[string]interface{}{
- "args": args,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stp.usage()
- return
- }
-
- err = stp.checkFlag()
- if err != nil {
- rlog.Error("Check Flag Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stp.usage()
- return
- }
-
- //p, err := rocketmq.NewProducer(&rocketmq.ProducerConfig{
- // ClientConfig: rocketmq.ClientConfig{GroupID: stp.groupID, NameServer: stp.nameSrv},
- //})
- //if err != nil {
- // fmt.Printf("new consumer error:%s\n", err)
- // return
- //}
- //
- //err = p.Start()
- //if err != nil {
- // fmt.Printf("start consumer error:%s\n", err)
- // return
- //}
- //defer p.Shutdown()
- //
- //stp.p = p
- stp.stableTest.run()
-}
-
-func (stp *stableTestProducer) sendMessage() {
- //r, err := stp.p.SendMessageSync(&rocketmq.Message{Topic: stp.topic, Body: buildMsg(stp.bodySize)})
- //if err == nil {
- // fmt.Printf("send result:%+v\n", r)
- // return
- //}
- //fmt.Printf("send message error:%s", err)
-}
-
-type stableTestConsumer struct {
- *stableTest
- expression string
-
- //c rocketmq.PullConsumer
- offsets map[int]int64
-}
-
-func (stc *stableTestConsumer) buildFlags(name string) {
- stc.stableTest.buildFlags(name)
- stc.flags.StringVar(&stc.expression, "e", "*", "expression")
-}
-
-func (stc *stableTestConsumer) checkFlag() error {
- err := stc.stableTest.checkFlag()
- if err != nil {
- return err
- }
-
- if stc.expression == "" {
- return errors.ErrEmptyExpression
- }
- return nil
-}
-
-func (stc *stableTestConsumer) usage() {
- stc.flags.Usage()
-}
-
-func (stc *stableTestConsumer) run(args []string) {
- err := stc.flags.Parse(args)
- if err != nil {
- rlog.Error("Parse Args Error", map[string]interface{}{
- "args": args,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stc.usage()
- return
- }
-
- err = stc.checkFlag()
- if err != nil {
- rlog.Error("Check Flag Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stc.usage()
- return
- }
- //
- //c, err := rocketmq.NewPullConsumer(&rocketmq.PullConsumerConfig{
- // ClientConfig: rocketmq.ClientConfig{GroupID: stc.groupID, NameServer: stc.nameSrv},
- //})
- //if err != nil {
- // fmt.Printf("new pull consumer error:%s\n", err)
- // return
- //}
- //
- //err = c.Start()
- //if err != nil {
- // fmt.Printf("start consumer error:%s\n", err)
- // return
- //}
- //defer c.Shutdown()
- //
- //stc.c = c
- stc.stableTest.run()
-}
-
-func (stc *stableTestConsumer) pullMessage() {
- //mqs := stc.c.FetchSubscriptionMessageQueues(stc.topic)
- //
- //for _, mq := range mqs {
- // offset := stc.offsets[mq.ID]
- // pr := stc.c.Pull(mq, stc.expression, offset, 32)
- //fmt.Printf("pull from %s, offset:%d, count:%+v\n", mq.String(), offset, len(pr.Messages))
- //
- //switch pr.Status {
- //case rocketmq.PullNoNewMsg:
- // stc.offsets[mq.ID] = 0 // pull from the begin
- //case rocketmq.PullFound:
- // fallthrough
- //case rocketmq.PullNoMatchedMsg:
- // fallthrough
- //case rocketmq.PullOffsetIllegal:
- // stc.offsets[mq.ID] = pr.NextBeginOffset
- //case rocketmq.PullBrokerTimeout:
- // fmt.Println("broker timeout occur")
- //}
- //}
-}
-
-func init() {
- // consumer
- name := "stableTestProducer"
- p := &stableTestProducer{stableTest: &stableTest{}}
- p.buildFlags(name)
- p.op = p.sendMessage
- registerCommand(name, p)
-
- // consumer
- name = "stableTestConsumer"
- c := &stableTestConsumer{stableTest: &stableTest{}, offsets: map[int]int64{}}
- c.buildFlags(name)
- c.op = c.pullMessage
- registerCommand(name, c)
-}
diff --git a/config.go b/config.go
deleted file mode 100644
index 8f1592c..0000000
--- a/config.go
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-package rocketmq
diff --git a/consumer/consumer.go b/consumer/consumer.go
deleted file mode 100644
index 8056b22..0000000
--- a/consumer/consumer.go
+++ /dev/null
@@ -1,1063 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "sort"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- jsoniter "github.com/json-iterator/go"
-
- "github.com/tidwall/gjson"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- // Delay some time when exception error
- _PullDelayTimeWhenError = 3 * time.Second
-
- // Flow control interval
- _PullDelayTimeWhenFlowControl = 50 * time.Millisecond
-
- // Delay some time when suspend pull service
- _PullDelayTimeWhenSuspend = 30 * time.Second
-
- // Long polling mode, the Consumer connection max suspend time
- _BrokerSuspendMaxTime = 20 * time.Second
-
- // Long polling mode, the Consumer connection timeout (must greater than _BrokerSuspendMaxTime)
- _ConsumerTimeoutWhenSuspend = 30 * time.Second
-
- // Offset persistent interval for consumer
- _PersistConsumerOffsetInterval = 5 * time.Second
-)
-
-type ConsumeType string
-
-const (
- _PullConsume = ConsumeType("CONSUME_ACTIVELY")
- _PushConsume = ConsumeType("CONSUME_PASSIVELY")
-
- _SubAll = "*"
-)
-
-// Message model defines the way how messages are delivered to each consumer clients.
-// </p>
-//
-// RocketMQ supports two message models: clustering and broadcasting. If clustering is set, consumer clients with
-// the same {@link #ConsumerGroup} would only consume shards of the messages subscribed, which achieves load
-// balances; Conversely, if the broadcasting is set, each consumer client will consume all subscribed messages
-// separately.
-// </p>
-//
-// This field defaults to clustering.
-type MessageModel int
-
-const (
- BroadCasting MessageModel = iota
- Clustering
-)
-
-func (mode MessageModel) String() string {
- switch mode {
- case BroadCasting:
- return "BroadCasting"
- case Clustering:
- return "Clustering"
- default:
- return "Unknown"
- }
-}
-
-// Consuming point on consumer booting.
-// </p>
-//
-// There are three consuming points:
-// <ul>
-// <li>
-// <code>CONSUME_FROM_LAST_OFFSET</code>: consumer clients pick up where it stopped previously.
-// If it were a newly booting up consumer client, according aging of the consumer group, there are two
-// cases:
-// <ol>
-// <li>
-// if the consumer group is created so recently that the earliest message being subscribed has yet
-// expired, which means the consumer group represents a lately launched business, consuming will
-// start from the very beginning;
-// </li>
-// <li>
-// if the earliest message being subscribed has expired, consuming will start from the latest
-// messages, meaning messages born prior to the booting timestamp would be ignored.
-// </li>
-// </ol>
-// </li>
-// <li>
-// <code>CONSUME_FROM_FIRST_OFFSET</code>: Consumer client will start from earliest messages available.
-// </li>
-// <li>
-// <code>CONSUME_FROM_TIMESTAMP</code>: Consumer client will start from specified timestamp, which means
-// messages born prior to {@link #consumeTimestamp} will be ignored
-// </li>
-// </ul>
-type ConsumeFromWhere int
-
-const (
- ConsumeFromLastOffset ConsumeFromWhere = iota
- ConsumeFromFirstOffset
- ConsumeFromTimestamp
-)
-
-type ExpressionType string
-
-const (
- /**
- * <ul>
- * Keywords:
- * <li>{@code AND, OR, NOT, BETWEEN, IN, TRUE, FALSE, IS, NULL}</li>
- * </ul>
- * <p/>
- * <ul>
- * Data type:
- * <li>Boolean, like: TRUE, FALSE</li>
- * <li>String, like: 'abc'</li>
- * <li>Decimal, like: 123</li>
- * <li>Float number, like: 3.1415</li>
- * </ul>
- * <p/>
- * <ul>
- * Grammar:
- * <li>{@code AND, OR}</li>
- * <li>{@code >, >=, <, <=, =}</li>
- * <li>{@code BETWEEN A AND B}, equals to {@code >=A AND <=B}</li>
- * <li>{@code NOT BETWEEN A AND B}, equals to {@code >B OR <A}</li>
- * <li>{@code IN ('a', 'b')}, equals to {@code ='a' OR ='b'}, this operation only support String type.</li>
- * <li>{@code IS NULL}, {@code IS NOT NULL}, check parameter whether is null, or not.</li>
- * <li>{@code =TRUE}, {@code =FALSE}, check parameter whether is true, or false.</li>
- * </ul>
- * <p/>
- * <p>
- * Example:
- * (a > 10 AND a < 100) OR (b IS NOT NULL AND b=TRUE)
- * </p>
- */
- SQL92 = ExpressionType("SQL92")
-
- /**
- * Only support or operation such as
- * "tag1 || tag2 || tag3", <br>
- * If null or * expression, meaning subscribe all.
- */
- TAG = ExpressionType("TAG")
-)
-
-func IsTagType(exp string) bool {
- if exp == "" || exp == "TAG" {
- return true
- }
- return false
-}
-
-type MessageSelector struct {
- Type ExpressionType
- Expression string
-}
-
-type ConsumeResult int
-
-const (
- ConsumeSuccess ConsumeResult = iota
- ConsumeRetryLater
- Commit
- Rollback
- SuspendCurrentQueueAMoment
-)
-
-type ConsumeResultHolder struct {
- ConsumeResult
-}
-
-type ConsumerReturn int
-
-const (
- SuccessReturn ConsumerReturn = iota
- ExceptionReturn
- NullReturn
- TimeoutReturn
- FailedReturn
-)
-
-type PullRequest struct {
- consumerGroup string
- mq *primitive.MessageQueue
- pq *processQueue
- nextOffset int64
- lockedFirst bool
-}
-
-func (pr *PullRequest) String() string {
- return fmt.Sprintf("[ConsumerGroup: %s, Topic: %s, MessageQueue: %d]",
- pr.consumerGroup, pr.mq.Topic, pr.mq.QueueId)
-}
-
-type defaultConsumer struct {
- /**
- * Consumers of the same role is required to have exactly same subscriptions and consumerGroup to correctly achieve
- * load balance. It's required and needs to be globally unique.
- * </p>
- *
- * See <a href="http://rocketmq.apache.org/docs/core-concept/">here</a> for further discussion.
- */
- consumerGroup string
- model MessageModel
- allocate func(string, string, []*primitive.MessageQueue, []string) []*primitive.MessageQueue
- unitMode bool
- consumeOrderly bool
- fromWhere ConsumeFromWhere
- consumerStartTimestamp int64
-
- cType ConsumeType
- client internal.RMQClient
- mqChanged func(topic string, mqAll, mqDivided []*primitive.MessageQueue)
- state int32
- pause bool
- once sync.Once
- option consumerOptions
- // key: primitive.MessageQueue
- // value: *processQueue
- processQueueTable sync.Map
-
- // key: topic(string)
- // value: map[int]*primitive.MessageQueue
- topicSubscribeInfoTable sync.Map
-
- // key: topic
- // value: *SubscriptionData
- subscriptionDataTable sync.Map
- storage OffsetStore
- // chan for push consumer
- prCh chan PullRequest
-
- pullFromWhichNodeTable sync.Map
-
- stat *StatsManager
-}
-
-func (dc *defaultConsumer) start() error {
- if dc.model == Clustering {
- // set retry topic
- retryTopic := internal.GetRetryTopic(dc.consumerGroup)
- sub := buildSubscriptionData(retryTopic, MessageSelector{TAG, _SubAll})
- dc.subscriptionDataTable.Store(retryTopic, sub)
- }
-
- if dc.model == Clustering {
- dc.option.ChangeInstanceNameToPID()
- dc.storage = NewRemoteOffsetStore(dc.consumerGroup, dc.client, dc.client.GetNameSrv())
- } else {
- dc.storage = NewLocalFileOffsetStore(dc.consumerGroup, dc.client.ClientID())
- }
-
- dc.client.Start()
- atomic.StoreInt32(&dc.state, int32(internal.StateRunning))
- dc.consumerStartTimestamp = time.Now().UnixNano() / int64(time.Millisecond)
- dc.stat = NewStatsManager()
- return nil
-}
-
-func (dc *defaultConsumer) shutdown() error {
- atomic.StoreInt32(&dc.state, int32(internal.StateShutdown))
-
- mqs := make([]*primitive.MessageQueue, 0)
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- k := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- pq.WithDropped(true)
- // close msg channel using RWMutex to make sure no data was writing
- pq.mutex.Lock()
- close(pq.msgCh)
- pq.mutex.Unlock()
- mqs = append(mqs, &k)
- return true
- })
- dc.stat.ShutDownStat()
- dc.storage.persist(mqs)
- dc.client.Shutdown()
- return nil
-}
-
-func (dc *defaultConsumer) persistConsumerOffset() error {
- err := dc.makeSureStateOK()
- if err != nil {
- return err
- }
- mqs := make([]*primitive.MessageQueue, 0)
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- k := key.(primitive.MessageQueue)
- mqs = append(mqs, &k)
- return true
- })
- dc.storage.persist(mqs)
- return nil
-}
-
-func (dc *defaultConsumer) updateOffset(queue *primitive.MessageQueue, offset int64) error {
- dc.storage.update(queue, offset, false)
- return nil
-}
-
-func (dc *defaultConsumer) subscriptionAutomatically(topic string) {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if !exist {
- s := MessageSelector{
- Expression: _SubAll,
- }
- dc.subscriptionDataTable.Store(topic, buildSubscriptionData(topic, s))
- }
-}
-
-func (dc *defaultConsumer) updateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if exist {
- dc.topicSubscribeInfoTable.Store(topic, mqs)
- }
-}
-
-func (dc *defaultConsumer) isSubscribeTopicNeedUpdate(topic string) bool {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if !exist {
- return false
- }
- _, exist = dc.topicSubscribeInfoTable.Load(topic)
- return !exist
-}
-
-func (dc *defaultConsumer) doBalance() {
- dc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- topic := key.(string)
- v, exist := dc.topicSubscribeInfoTable.Load(topic)
- if !exist {
- rlog.Warning("do balance in group failed, the topic does not exist", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- })
- return true
- }
- mqs := v.([]*primitive.MessageQueue)
- switch dc.model {
- case BroadCasting:
- changed := dc.updateProcessQueueTable(topic, mqs)
- if changed {
- dc.mqChanged(topic, mqs, mqs)
- rlog.Debug("MessageQueue changed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- rlog.LogKeyMessageQueue: fmt.Sprintf("%v", mqs),
- })
- }
- case Clustering:
- cidAll := dc.findConsumerList(topic)
- if cidAll == nil {
- rlog.Warning("do balance in group failed, get consumer id list failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- })
- return true
- }
- mqAll := make([]*primitive.MessageQueue, len(mqs))
- copy(mqAll, mqs)
- sort.Strings(cidAll)
- sort.SliceStable(mqAll, func(i, j int) bool {
- v := strings.Compare(mqAll[i].Topic, mqAll[j].Topic)
- if v != 0 {
- return v < 0
- }
-
- v = strings.Compare(mqAll[i].BrokerName, mqAll[j].BrokerName)
- if v != 0 {
- return v < 0
- }
- return (mqAll[i].QueueId - mqAll[j].QueueId) < 0
- })
- allocateResult := dc.allocate(dc.consumerGroup, dc.client.ClientID(), mqAll, cidAll)
- changed := dc.updateProcessQueueTable(topic, allocateResult)
- if changed {
- dc.mqChanged(topic, mqAll, allocateResult)
- rlog.Debug("MessageQueue do balance done", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- "clientID": dc.client.ClientID(),
- "mqAllSize": len(mqAll),
- "cidAllSize": len(cidAll),
- "rebalanceResultSize": len(allocateResult),
- "rebalanceResultSet": allocateResult,
- })
- }
- }
- return true
- })
-}
-
-func (dc *defaultConsumer) SubscriptionDataList() []*internal.SubscriptionData {
- result := make([]*internal.SubscriptionData, 0)
- dc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- result = append(result, value.(*internal.SubscriptionData))
- return true
- })
- return result
-}
-
-func (dc *defaultConsumer) makeSureStateOK() error {
- if atomic.LoadInt32(&dc.state) != int32(internal.StateRunning) {
- return fmt.Errorf("state not running, actually: %v", dc.state)
- }
- return nil
-}
-
-type lockBatchRequestBody struct {
- ConsumerGroup string `json:"consumerGroup"`
- ClientId string `json:"clientId"`
- MQs []*primitive.MessageQueue `json:"mqSet"`
-}
-
-func (dc *defaultConsumer) lock(mq *primitive.MessageQueue) bool {
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, internal.MasterId, true)
-
- if brokerResult == nil {
- return false
- }
-
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: []*primitive.MessageQueue{mq},
- }
- lockedMQ := dc.doLock(brokerResult.BrokerAddr, body)
- var lockOK bool
- for idx := range lockedMQ {
- _mq := lockedMQ[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(true)
- pq.UpdateLastConsumeTime()
- pq.UpdateLastLockTime()
- }
- if _mq == *mq {
- lockOK = true
- }
- }
- fields := map[string]interface{}{
- "lockOK": lockOK,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- }
- if lockOK {
- rlog.Debug("lock MessageQueue", fields)
- } else {
- rlog.Info("lock MessageQueue", fields)
- }
- return lockOK
-}
-
-func (dc *defaultConsumer) unlock(mq *primitive.MessageQueue, oneway bool) {
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, internal.MasterId, true)
-
- if brokerResult == nil {
- return
- }
-
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: []*primitive.MessageQueue{mq},
- }
- dc.doUnlock(brokerResult.BrokerAddr, body, oneway)
- rlog.Info("unlock MessageQueue", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- "clientID": dc.client.ClientID(),
- rlog.LogKeyMessageQueue: mq.String(),
- })
-}
-
-func (dc *defaultConsumer) lockAll() {
- mqMapSet := dc.buildProcessQueueTableByBrokerName()
- for broker, mqs := range mqMapSet {
- if len(mqs) == 0 {
- continue
- }
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(broker, internal.MasterId, true)
- if brokerResult == nil {
- continue
- }
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: mqs,
- }
- lockedMQ := dc.doLock(brokerResult.BrokerAddr, body)
- set := make(map[primitive.MessageQueue]bool)
- for idx := range lockedMQ {
- _mq := lockedMQ[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(true)
- pq.UpdateLastConsumeTime()
- }
- set[_mq] = true
- }
- for idx := range mqs {
- _mq := mqs[idx]
- if !set[*_mq] {
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(false)
- pq.UpdateLastLockTime()
- rlog.Info("lock MessageQueue", map[string]interface{}{
- "lockOK": false,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: _mq.String(),
- })
- }
- }
- }
- }
-}
-
-func (dc *defaultConsumer) unlockAll(oneway bool) {
- mqMapSet := dc.buildProcessQueueTableByBrokerName()
- for broker, mqs := range mqMapSet {
- if len(mqs) == 0 {
- continue
- }
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(broker, internal.MasterId, true)
- if brokerResult == nil {
- continue
- }
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: mqs,
- }
- dc.doUnlock(brokerResult.BrokerAddr, body, oneway)
- for idx := range mqs {
- _mq := mqs[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- rlog.Info("lock MessageQueue", map[string]interface{}{
- "lockOK": false,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: _mq.String(),
- })
- v.(*processQueue).WithLock(false)
- }
- }
- }
-}
-
-func (dc *defaultConsumer) doLock(addr string, body *lockBatchRequestBody) []primitive.MessageQueue {
- data, _ := jsoniter.Marshal(body)
- request := remote.NewRemotingCommand(internal.ReqLockBatchMQ, nil, data)
- response, err := dc.client.InvokeSync(context.Background(), addr, request, 1*time.Second)
- if err != nil {
- rlog.Error("lock MessageQueue to broker invoke error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- lockOKMQSet := struct {
- MQs []primitive.MessageQueue `json:"lockOKMQSet"`
- }{}
- if len(response.Body) == 0 {
- return nil
- }
- err = jsoniter.Unmarshal(response.Body, &lockOKMQSet)
- if err != nil {
- rlog.Error("Unmarshal lock mq body error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- return lockOKMQSet.MQs
-}
-
-func (dc *defaultConsumer) doUnlock(addr string, body *lockBatchRequestBody, oneway bool) {
- data, _ := jsoniter.Marshal(body)
- request := remote.NewRemotingCommand(internal.ReqUnlockBatchMQ, nil, data)
- if oneway {
- err := dc.client.InvokeOneWay(context.Background(), addr, request, 3*time.Second)
- if err != nil {
- rlog.Error("lock MessageQueue to broker invoke oneway error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- }
- } else {
- response, err := dc.client.InvokeSync(context.Background(), addr, request, 1*time.Second)
- rlog.Error("lock MessageQueue to broker invoke error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- if response.Code != internal.ResSuccess {
- // TODO error
- }
- }
-}
-
-func (dc *defaultConsumer) buildProcessQueueTableByBrokerName() map[string][]*primitive.MessageQueue {
- result := make(map[string][]*primitive.MessageQueue, 0)
-
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- mqs, exist := result[mq.BrokerName]
- if !exist {
- mqs = make([]*primitive.MessageQueue, 0)
- }
- mqs = append(mqs, &mq)
- result[mq.BrokerName] = mqs
- return true
- })
-
- return result
-}
-
-func (dc *defaultConsumer) updateProcessQueueTable(topic string, mqs []*primitive.MessageQueue) bool {
- var changed bool
- mqSet := make(map[primitive.MessageQueue]bool)
- for idx := range mqs {
- mqSet[*mqs[idx]] = true
- }
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- if mq.Topic == topic {
- if !mqSet[mq] {
- pq.WithDropped(true)
- if dc.removeUnnecessaryMessageQueue(&mq, pq) {
- dc.processQueueTable.Delete(key)
- changed = true
- rlog.Debug("remove unnecessary mq when updateProcessQueueTable", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- } else if pq.isPullExpired() && dc.cType == _PushConsume {
- pq.WithDropped(true)
- if dc.removeUnnecessaryMessageQueue(&mq, pq) {
- dc.processQueueTable.Delete(key)
- changed = true
- rlog.Debug("remove unnecessary mq because pull was paused, prepare to fix it", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- }
- }
- return true
- })
-
- if dc.cType == _PushConsume {
- for item := range mqSet {
- // BUG: the mq will send to channel, if not copy once, the next iter will modify the mq in the channel.
- mq := item
- _, exist := dc.processQueueTable.Load(mq)
- if exist {
- continue
- }
- if dc.consumeOrderly && !dc.lock(&mq) {
- rlog.Warning("do defaultConsumer, add a new mq failed, because lock failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- continue
- }
- dc.storage.remove(&mq)
- nextOffset := dc.computePullFromWhere(&mq)
- if nextOffset >= 0 {
- _, exist := dc.processQueueTable.Load(mq)
- if exist {
- rlog.Debug("do defaultConsumer, mq already exist", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- } else {
- rlog.Debug("do defaultConsumer, add a new mq", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pq := newProcessQueue(dc.consumeOrderly)
- dc.processQueueTable.Store(mq, pq)
- pr := PullRequest{
- consumerGroup: dc.consumerGroup,
- mq: &mq,
- pq: pq,
- nextOffset: nextOffset,
- }
- dc.prCh <- pr
- changed = true
- }
- } else {
- rlog.Warning("do defaultConsumer, add a new mq failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- }
- }
-
- return changed
-}
-
-func (dc *defaultConsumer) removeUnnecessaryMessageQueue(mq *primitive.MessageQueue, pq *processQueue) bool {
- dc.storage.persist([]*primitive.MessageQueue{mq})
- dc.storage.remove(mq)
- return true
-}
-
-func (dc *defaultConsumer) computePullFromWhere(mq *primitive.MessageQueue) int64 {
- if dc.cType == _PullConsume {
- return 0
- }
- var result = int64(-1)
- lastOffset := dc.storage.read(mq, _ReadFromStore)
- if lastOffset >= 0 {
- result = lastOffset
- } else {
- switch dc.option.FromWhere {
- case ConsumeFromLastOffset:
- if lastOffset == -1 {
- if strings.HasPrefix(mq.Topic, internal.RetryGroupTopicPrefix) {
- result = 0
- } else {
- lastOffset, err := dc.queryMaxOffset(mq)
- if err == nil {
- result = lastOffset
- } else {
- rlog.Warning("query max offset error", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- rlog.LogKeyUnderlayError: err,
- })
- }
- }
- } else {
- result = -1
- }
- case ConsumeFromFirstOffset:
- if lastOffset == -1 {
- result = 0
- }
- case ConsumeFromTimestamp:
- if lastOffset == -1 {
- if strings.HasPrefix(mq.Topic, internal.RetryGroupTopicPrefix) {
- lastOffset, err := dc.queryMaxOffset(mq)
- if err == nil {
- result = lastOffset
- } else {
- result = -1
- rlog.Warning("query max offset error", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- rlog.LogKeyUnderlayError: err,
- })
- }
- } else {
- t, err := time.Parse("20060102150405", dc.option.ConsumeTimestamp)
- if err != nil {
- result = -1
- } else {
- lastOffset, err := dc.searchOffsetByTimestamp(mq, t.Unix()*1000)
- if err != nil {
- result = -1
- } else {
- result = lastOffset
- }
- }
- }
- }
- default:
- }
- }
- return result
-}
-
-func (dc *defaultConsumer) pullInner(ctx context.Context, queue *primitive.MessageQueue, data *internal.SubscriptionData,
- offset int64, numbers int, sysFlag int32, commitOffsetValue int64) (*primitive.PullResult, error) {
-
- brokerResult := dc.tryFindBroker(queue)
- if brokerResult == nil {
- rlog.Warning("no broker found for mq", map[string]interface{}{
- rlog.LogKeyMessageQueue: queue,
- })
- return nil, errors.ErrBrokerNotFound
- }
-
- if brokerResult.Slave {
- sysFlag = clearCommitOffsetFlag(sysFlag)
- }
-
- if (data.ExpType == string(TAG)) && brokerResult.BrokerVersion < internal.V4_1_0 {
- return nil, fmt.Errorf("the broker [%s, %v] does not upgrade to support for filter message by %v",
- queue.BrokerName, brokerResult.BrokerVersion, data.ExpType)
- }
-
- pullRequest := &internal.PullMessageRequestHeader{
- ConsumerGroup: dc.consumerGroup,
- Topic: queue.Topic,
- QueueId: int32(queue.QueueId),
- QueueOffset: offset,
- MaxMsgNums: int32(numbers),
- SysFlag: sysFlag,
- CommitOffset: commitOffsetValue,
- // TODO: 和java对齐
- SuspendTimeoutMillis: _BrokerSuspendMaxTime,
- SubExpression: data.SubString,
- // TODO: add subversion
- ExpressionType: string(data.ExpType),
- }
-
- if data.ExpType == string(TAG) {
- pullRequest.SubVersion = 0
- } else {
- pullRequest.SubVersion = data.SubVersion
- }
-
- // TODO: add computPullFromWhichFilterServer
-
- return dc.client.PullMessage(context.Background(), brokerResult.BrokerAddr, pullRequest)
-}
-
-func (dc *defaultConsumer) processPullResult(mq *primitive.MessageQueue, result *primitive.PullResult, data *internal.SubscriptionData) {
-
- dc.updatePullFromWhichNode(mq, result.SuggestWhichBrokerId)
-
- switch result.Status {
- case primitive.PullFound:
- result.SetMessageExts(primitive.DecodeMessage(result.GetBody()))
- msgs := result.GetMessageExts()
-
- // filter message according to tags
- msgListFilterAgain := msgs
- if data.Tags.Len() > 0 && data.ClassFilterMode {
- msgListFilterAgain = make([]*primitive.MessageExt, 0)
- for _, msg := range msgs {
- _, exist := data.Tags.Contains(msg.GetTags())
- if exist {
- msgListFilterAgain = append(msgListFilterAgain, msg)
- }
- }
- }
-
- // TODO: add filter message hook
- for _, msg := range msgListFilterAgain {
- traFlag, _ := strconv.ParseBool(msg.GetProperty(primitive.PropertyTransactionPrepared))
- if traFlag {
- msg.TransactionId = msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
- }
-
- msg.WithProperty(primitive.PropertyMinOffset, strconv.FormatInt(result.MinOffset, 10))
- msg.WithProperty(primitive.PropertyMaxOffset, strconv.FormatInt(result.MaxOffset, 10))
- }
-
- result.SetMessageExts(msgListFilterAgain)
- }
-}
-
-func (dc *defaultConsumer) findConsumerList(topic string) []string {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByTopic(topic)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByTopic(topic)
- }
-
- if brokerAddr != "" {
- req := &internal.GetConsumerListRequestHeader{
- ConsumerGroup: dc.consumerGroup,
- }
- cmd := remote.NewRemotingCommand(internal.ReqGetConsumerListByGroup, req, nil)
- res, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second) // TODO 超时机制有问题
- if err != nil {
- rlog.Error("get consumer list of group from broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyBroker: brokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- result := gjson.ParseBytes(res.Body)
- list := make([]string, 0)
- arr := result.Get("consumerIdList").Array()
- for idx := range arr {
- list = append(list, arr[idx].String())
- }
- return list
- }
- return nil
-}
-
-func (dc *defaultConsumer) sendBack(msg *primitive.MessageExt, level int) error {
- return nil
-}
-
-// QueryMaxOffset with specific queueId and topic
-func (dc *defaultConsumer) queryMaxOffset(mq *primitive.MessageQueue) (int64, error) {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- }
- if brokerAddr == "" {
- return -1, fmt.Errorf("the broker [%s] does not exist", mq.BrokerName)
- }
-
- request := &internal.GetMaxOffsetRequestHeader{
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqGetMaxOffset, request, nil)
- response, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
-
- return strconv.ParseInt(response.ExtFields["offset"], 10, 64)
-}
-
-func (dc *defaultConsumer) queryOffset(mq *primitive.MessageQueue) int64 {
- return dc.storage.read(mq, _ReadMemoryThenStore)
-}
-
-// SearchOffsetByTimestamp with specific queueId and topic
-func (dc *defaultConsumer) searchOffsetByTimestamp(mq *primitive.MessageQueue, timestamp int64) (int64, error) {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- }
- if brokerAddr == "" {
- return -1, fmt.Errorf("the broker [%s] does not exist", mq.BrokerName)
- }
-
- request := &internal.SearchOffsetRequestHeader{
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- Timestamp: timestamp,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqSearchOffsetByTimestamp, request, nil)
- response, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
-
- return strconv.ParseInt(response.ExtFields["offset"], 10, 64)
-}
-
-func buildSubscriptionData(topic string, selector MessageSelector) *internal.SubscriptionData {
- subData := &internal.SubscriptionData{
- Topic: topic,
- SubString: selector.Expression,
- ExpType: string(selector.Type),
- }
- subData.SubVersion = time.Now().UnixNano()
-
- if selector.Type != "" && selector.Type != TAG {
- return subData
- }
-
- if selector.Expression == "" || selector.Expression == _SubAll {
- subData.ExpType = string(TAG)
- subData.SubString = _SubAll
- } else {
- tags := strings.Split(selector.Expression, "||")
- subData.Tags = utils.NewSet()
- subData.Codes = utils.NewSet()
- for idx := range tags {
- trimString := strings.Trim(tags[idx], " ")
- if trimString != "" {
- if _, ok := subData.Tags.Contains(trimString); !ok {
- subData.Tags.AddKV(trimString, trimString)
- }
- hCode := utils.HashString(trimString)
- v := strconv.Itoa(hCode)
- if _, ok := subData.Codes.Contains(v); !ok {
- subData.Codes.AddKV(v, v)
- }
- }
- }
- }
- return subData
-}
-
-func buildSysFlag(commitOffset, suspend, subscription, classFilter bool) int32 {
- var flag int32 = 0
- if commitOffset {
- flag |= 0x1 << 0
- }
-
- if suspend {
- flag |= 0x1 << 1
- }
-
- if subscription {
- flag |= 0x1 << 2
- }
-
- if classFilter {
- flag |= 0x1 << 3
- }
-
- return flag
-}
-
-func clearCommitOffsetFlag(sysFlag int32) int32 {
- return sysFlag & (^0x1 << 0)
-}
-
-func (dc *defaultConsumer) tryFindBroker(mq *primitive.MessageQueue) *internal.FindBrokerResult {
- result := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, dc.recalculatePullFromWhichNode(mq), false)
- if result != nil {
- return result
- }
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- return dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, dc.recalculatePullFromWhichNode(mq), false)
-}
-
-func (dc *defaultConsumer) updatePullFromWhichNode(mq *primitive.MessageQueue, brokerId int64) {
- dc.pullFromWhichNodeTable.Store(*mq, brokerId)
-}
-
-func (dc *defaultConsumer) recalculatePullFromWhichNode(mq *primitive.MessageQueue) int64 {
- v, exist := dc.pullFromWhichNodeTable.Load(*mq)
- if exist {
- return v.(int64)
- }
- return internal.MasterId
-}
diff --git a/consumer/consumer_test.go b/consumer/consumer_test.go
deleted file mode 100644
index 12ccd18..0000000
--- a/consumer/consumer_test.go
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "sync"
- "testing"
- "time"
-
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func TestParseTimestamp(t *testing.T) {
- layout := "20060102150405"
- timestamp, err := time.ParseInLocation(layout, "20190430193409", time.UTC)
- assert.Nil(t, err)
- assert.Equal(t, int64(1556652849), timestamp.Unix())
-}
-
-func TestDoRebalance(t *testing.T) {
- Convey("Given a defaultConsumer", t, func() {
- dc := &defaultConsumer{
- model: Clustering,
- }
-
- topic := "test"
- broker := "127.0.0.1:8889"
- clientID := "clientID"
- mqs := []*primitive.MessageQueue{
- {
- Topic: topic,
- BrokerName: "",
- QueueId: 0,
- },
- {
- Topic: topic,
- BrokerName: "",
- QueueId: 1,
- },
- }
- dc.topicSubscribeInfoTable.Store(topic, mqs)
- sub := &internal.SubscriptionData{}
- dc.subscriptionDataTable.Store(topic, sub)
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- namesrvCli := internal.NewMockNamesrvs(ctrl)
- namesrvCli.EXPECT().FindBrokerAddrByTopic(gomock.Any()).Return(broker)
-
- rmqCli := internal.NewMockRMQClient(ctrl)
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- Body: []byte("{\"consumerIdList\": [\"a1\", \"a2\", \"a3\"] }"),
- }, nil)
- rmqCli.EXPECT().ClientID().Return(clientID)
- rmqCli.SetNameSrv(namesrvCli)
-
- dc.client = rmqCli
-
- var wg sync.WaitGroup
- wg.Add(1)
- dc.allocate = func(cg string, clientID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- assert.Equal(t, cidAll, []string{"a1", "a2", "a3"})
- wg.Done()
- return nil
- }
-
- dc.doBalance()
-
- wg.Wait()
- })
-}
-
-func TestComputePullFromWhere(t *testing.T) {
- Convey("Given a defaultConsumer", t, func() {
- dc := &defaultConsumer{
- model: Clustering,
- cType: _PushConsume,
- }
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- offsetStore := NewMockOffsetStore(ctrl)
- dc.storage = offsetStore
-
- mq := &primitive.MessageQueue{
- Topic: "test",
- }
-
- namesrvCli := internal.NewMockNamesrvs(ctrl)
-
- rmqCli := internal.NewMockRMQClient(ctrl)
- dc.client = rmqCli
- rmqCli.SetNameSrv(namesrvCli)
-
- Convey("get effective offset", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(10))
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(10), res)
- })
-
- Convey("ConsumeFromLastOffset for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromLastOffset
-
- broker := "a"
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return(broker)
-
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- ExtFields: map[string]string{
- "offset": "20",
- },
- }, nil)
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(20), res)
- })
-
- Convey("ConsumeFromFirstOffset for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromFirstOffset
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(0), res)
- })
-
- Convey("ConsumeFromTimestamp for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromTimestamp
-
- dc.option.ConsumeTimestamp = "20060102150405"
-
- broker := "a"
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return(broker)
-
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- ExtFields: map[string]string{
- "offset": "30",
- },
- }, nil)
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(30), res)
- })
-
- })
-}
diff --git a/consumer/interceptor.go b/consumer/interceptor.go
deleted file mode 100644
index 05ff94a..0000000
--- a/consumer/interceptor.go
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// WithTrace support rocketmq trace: https://github.com/apache/rocketmq/wiki/RIP-6-Message-Trace.
-func WithTrace(traceCfg *primitive.TraceConfig) Option {
- return func(options *consumerOptions) {
-
- ori := options.Interceptors
- options.Interceptors = make([]primitive.Interceptor, 0)
- options.Interceptors = append(options.Interceptors, newTraceInterceptor(traceCfg))
- options.Interceptors = append(options.Interceptors, ori...)
- }
-}
-
-func newTraceInterceptor(traceCfg *primitive.TraceConfig) primitive.Interceptor {
- dispatcher := internal.NewTraceDispatcher(traceCfg)
- if dispatcher != nil {
- dispatcher.Start()
- }
-
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- if dispatcher == nil {
- return fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- consumerCtx, exist := primitive.GetConsumerCtx(ctx)
- if !exist || len(consumerCtx.Msgs) == 0 {
- return next(ctx, req, reply)
- }
-
- beginT := time.Now()
- // before traceCtx
- traceCx := internal.TraceContext{
- RequestId: primitive.CreateUniqID(),
- TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
- TraceType: internal.SubBefore,
- GroupName: consumerCtx.ConsumerGroup,
- IsSuccess: true,
- }
- beans := make([]internal.TraceBean, 0)
- for _, msg := range consumerCtx.Msgs {
- if msg == nil {
- continue
- }
- regionID := msg.GetRegionID()
- traceOn := msg.IsTraceOn()
- if traceOn == "false" {
- continue
- }
- bean := internal.TraceBean{
- Topic: msg.Topic,
- MsgId: msg.MsgId,
- Tags: msg.GetTags(),
- Keys: msg.GetKeys(),
- StoreTime: msg.StoreTimestamp,
- BodyLength: int(msg.StoreSize),
- RetryTimes: int(msg.ReconsumeTimes),
- ClientHost: utils.LocalIP,
- StoreHost: utils.LocalIP,
- }
- beans = append(beans, bean)
- traceCx.RegionId = regionID
- }
- if len(beans) > 0 {
- traceCx.TraceBeans = beans
- traceCx.TimeStamp = time.Now().UnixNano() / int64(time.Millisecond)
- dispatcher.Append(traceCx)
- }
-
- err := next(ctx, req, reply)
-
- // after traceCtx
- costTime := time.Since(beginT).Nanoseconds() / int64(time.Millisecond)
- ctxType := consumerCtx.Properties[primitive.PropCtxType]
- afterCtx := internal.TraceContext{
- TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
-
- TraceType: internal.SubAfter,
- RegionId: traceCx.RegionId,
- GroupName: traceCx.GroupName,
- RequestId: traceCx.RequestId,
- IsSuccess: consumerCtx.Success,
- CostTime: costTime,
- TraceBeans: traceCx.TraceBeans,
- ContextCode: primitive.ConsumeReturnType(ctxType).Ordinal(),
- }
- dispatcher.Append(afterCtx)
- return err
- }
-}
diff --git a/consumer/lock.go b/consumer/lock.go
deleted file mode 100644
index 6fb17cd..0000000
--- a/consumer/lock.go
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "sync"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-type QueueLock struct {
- lockTable sync.Map
-}
-
-func newQueueLock() *QueueLock {
- return &QueueLock{}
-}
-
-func (ql QueueLock) fetchLock(queue primitive.MessageQueue) sync.Locker {
- v, _ := ql.lockTable.LoadOrStore(queue, new(sync.Mutex))
- return v.(*sync.Mutex)
-}
diff --git a/consumer/mock_offset_store.go b/consumer/mock_offset_store.go
deleted file mode 100644
index bac1cdb..0000000
--- a/consumer/mock_offset_store.go
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by MockGen. DO NOT EDIT.
-// Source: offset_store.go
-
-// Package consumer is a generated GoMock package.
-package consumer
-
-import (
- reflect "reflect"
-
- primitive "github.com/apache/rocketmq-client-go/v2/primitive"
- gomock "github.com/golang/mock/gomock"
-)
-
-// MockOffsetStore is a mock of OffsetStore interface
-type MockOffsetStore struct {
- ctrl *gomock.Controller
- recorder *MockOffsetStoreMockRecorder
-}
-
-// MockOffsetStoreMockRecorder is the mock recorder for MockOffsetStore
-type MockOffsetStoreMockRecorder struct {
- mock *MockOffsetStore
-}
-
-// NewMockOffsetStore creates a new mock instance
-func NewMockOffsetStore(ctrl *gomock.Controller) *MockOffsetStore {
- mock := &MockOffsetStore{ctrl: ctrl}
- mock.recorder = &MockOffsetStoreMockRecorder{mock}
- return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockOffsetStore) EXPECT() *MockOffsetStoreMockRecorder {
- return m.recorder
-}
-
-// persist mocks base method
-func (m *MockOffsetStore) persist(mqs []*primitive.MessageQueue) {
- m.ctrl.Call(m, "persist", mqs)
-}
-
-// persist indicates an expected call of persist
-func (mr *MockOffsetStoreMockRecorder) persist(mqs interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "persist", reflect.TypeOf((*MockOffsetStore)(nil).persist), mqs)
-}
-
-// remove mocks base method
-func (m *MockOffsetStore) remove(mq *primitive.MessageQueue) {
- m.ctrl.Call(m, "remove", mq)
-}
-
-// remove indicates an expected call of remove
-func (mr *MockOffsetStoreMockRecorder) remove(mq interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "remove", reflect.TypeOf((*MockOffsetStore)(nil).remove), mq)
-}
-
-// read mocks base method
-func (m *MockOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- ret := m.ctrl.Call(m, "read", mq, t)
- ret0, _ := ret[0].(int64)
- return ret0
-}
-
-// read indicates an expected call of read
-func (mr *MockOffsetStoreMockRecorder) read(mq, t interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "read", reflect.TypeOf((*MockOffsetStore)(nil).read), mq, t)
-}
-
-// update mocks base method
-func (m *MockOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- m.ctrl.Call(m, "update", mq, offset, increaseOnly)
-}
-
-// update indicates an expected call of update
-func (mr *MockOffsetStoreMockRecorder) update(mq, offset, increaseOnly interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "update", reflect.TypeOf((*MockOffsetStore)(nil).update), mq, offset, increaseOnly)
-}
diff --git a/consumer/offset_store.go b/consumer/offset_store.go
deleted file mode 100644
index 5ecfd14..0000000
--- a/consumer/offset_store.go
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "os"
- "path/filepath"
- "strconv"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- jsoniter "github.com/json-iterator/go"
-)
-
-type readType int
-
-const (
- _ReadFromMemory readType = iota
- _ReadFromStore
- _ReadMemoryThenStore
-)
-
-var (
- _LocalOffsetStorePath = os.Getenv("rocketmq.client.localOffsetStoreDir")
-)
-
-func init() {
- if _LocalOffsetStorePath == "" {
- _LocalOffsetStorePath = filepath.Join(os.Getenv("HOME"), ".rocketmq_client_go")
- }
-}
-
-//go:generate mockgen -source offset_store.go -destination mock_offset_store.go -self_package github.com/apache/rocketmq-client-go/v2/consumer --package consumer OffsetStore
-type OffsetStore interface {
- persist(mqs []*primitive.MessageQueue)
- remove(mq *primitive.MessageQueue)
- read(mq *primitive.MessageQueue, t readType) int64
- update(mq *primitive.MessageQueue, offset int64, increaseOnly bool)
-}
-
-type OffsetSerializeWrapper struct {
- OffsetTable map[MessageQueueKey]int64 `json:"offsetTable"`
-}
-
-type MessageQueueKey primitive.MessageQueue
-
-func (mq MessageQueueKey) MarshalText() (text []byte, err error) {
- repr := struct {
- Topic string `json:"topic"`
- BrokerName string `json:"brokerName"`
- QueueId int `json:"queueId"`
- }{
- Topic: mq.Topic,
- BrokerName: mq.BrokerName,
- QueueId: mq.QueueId,
- }
- text, err = jsoniter.Marshal(repr)
- return
-}
-
-func (mq *MessageQueueKey) UnmarshalText(text []byte) error {
- repr := struct {
- Topic string `json:"topic"`
- BrokerName string `json:"brokerName"`
- QueueId int `json:"queueId"`
- }{}
- err := jsoniter.Unmarshal(text, &repr)
- if err != nil {
- return err
- }
- mq.Topic = repr.Topic
- mq.QueueId = repr.QueueId
- mq.BrokerName = repr.BrokerName
-
- return nil
-}
-
-type localFileOffsetStore struct {
- group string
- path string
- OffsetTable *sync.Map // concurrent safe , map[MessageQueueKey]int64
- // mutex for offset file
- mutex sync.Mutex
-}
-
-func NewLocalFileOffsetStore(clientID, group string) OffsetStore {
- store := &localFileOffsetStore{
- group: group,
- path: filepath.Join(_LocalOffsetStorePath, clientID, group, "offset.json"),
- OffsetTable: new(sync.Map),
- }
- store.load()
- return store
-}
-
-func (local *localFileOffsetStore) load() {
- local.mutex.Lock()
- defer local.mutex.Unlock()
- data, err := utils.FileReadAll(local.path)
- if os.IsNotExist(err) {
- return
- }
- if err != nil {
- rlog.Info("read from local store error, try to use bak file", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- data, err = utils.FileReadAll(filepath.Join(local.path, ".bak"))
- }
- if err != nil {
- rlog.Info("read from local store bak file error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return
- }
- datas := make(map[MessageQueueKey]int64)
-
- wrapper := OffsetSerializeWrapper{
- OffsetTable: datas,
- }
-
- err = jsoniter.Unmarshal(data, &wrapper)
- if err != nil {
- rlog.Warning("unmarshal local offset error", map[string]interface{}{
- "local_path": local.path,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return
- }
-
- if datas != nil {
- for k, v := range datas {
- local.OffsetTable.Store(k, v)
- }
- }
-}
-
-func (local *localFileOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- switch t {
- case _ReadFromMemory, _ReadMemoryThenStore:
- off := readFromMemory(local.OffsetTable, mq)
- if off >= 0 || (off == -1 && t == _ReadFromMemory) {
- return off
- }
- fallthrough
- case _ReadFromStore:
- local.load()
- return readFromMemory(local.OffsetTable, mq)
- default:
-
- }
- return -1
-}
-
-func (local *localFileOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- local.mutex.Lock()
- defer local.mutex.Unlock()
- rlog.Debug("update offset", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- "new_offset": offset,
- })
- key := MessageQueueKey(*mq)
- localOffset, exist := local.OffsetTable.Load(key)
- if !exist {
- local.OffsetTable.Store(key, offset)
- return
- }
- if increaseOnly {
- if localOffset.(int64) < offset {
- local.OffsetTable.Store(key, offset)
- }
- } else {
- local.OffsetTable.Store(key, offset)
- }
-}
-
-func (local *localFileOffsetStore) persist(mqs []*primitive.MessageQueue) {
- if len(mqs) == 0 {
- return
- }
- local.mutex.Lock()
- defer local.mutex.Unlock()
-
- datas := make(map[MessageQueueKey]int64)
- local.OffsetTable.Range(func(key, value interface{}) bool {
- k := key.(MessageQueueKey)
- v := value.(int64)
- datas[k] = v
- return true
- })
-
- wrapper := OffsetSerializeWrapper{
- OffsetTable: datas,
- }
- data, _ := jsoniter.Marshal(wrapper)
- utils.CheckError(fmt.Sprintf("persist offset to %s", local.path), utils.WriteToFile(local.path, data))
-}
-
-func (local *localFileOffsetStore) remove(mq *primitive.MessageQueue) {
- // nothing to do
-}
-
-type remoteBrokerOffsetStore struct {
- group string
- OffsetTable map[primitive.MessageQueue]int64 `json:"OffsetTable"`
- client internal.RMQClient
- namesrv internal.Namesrvs
- mutex sync.RWMutex
-}
-
-func NewRemoteOffsetStore(group string, client internal.RMQClient, namesrv internal.Namesrvs) OffsetStore {
- return &remoteBrokerOffsetStore{
- group: group,
- client: client,
- namesrv: namesrv,
- OffsetTable: make(map[primitive.MessageQueue]int64),
- }
-}
-
-func (r *remoteBrokerOffsetStore) persist(mqs []*primitive.MessageQueue) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
- if len(mqs) == 0 {
- return
- }
-
- used := make(map[primitive.MessageQueue]struct{}, 0)
- for _, mq := range mqs {
- used[*mq] = struct{}{}
- }
-
- for mq, off := range r.OffsetTable {
- if _, ok := used[mq]; !ok {
- delete(r.OffsetTable, mq)
- continue
- }
- err := r.updateConsumeOffsetToBroker(r.group, mq, off)
- if err != nil {
- rlog.Warning("update offset to broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyUnderlayError: err.Error(),
- "offset": off,
- })
- } else {
- rlog.Info("update offset to broker success", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- "offset": off,
- })
- }
- }
-}
-
-func (r *remoteBrokerOffsetStore) remove(mq *primitive.MessageQueue) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
-
- delete(r.OffsetTable, *mq)
- rlog.Warning("delete mq from offset table", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq,
- })
-}
-
-func (r *remoteBrokerOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- r.mutex.RLock()
- switch t {
- case _ReadFromMemory, _ReadMemoryThenStore:
- off, exist := r.OffsetTable[*mq]
- if exist {
- r.mutex.RUnlock()
- return off
- }
- if t == _ReadFromMemory {
- r.mutex.RUnlock()
- return -1
- }
- fallthrough
- case _ReadFromStore:
- off, err := r.fetchConsumeOffsetFromBroker(r.group, mq)
- if err != nil {
- rlog.Error("fetch offset of mq from broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyUnderlayError: err,
- })
- r.mutex.RUnlock()
- return -1
- }
- rlog.Warning("fetch offset of mq from broker success", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- "offset": off,
- })
- r.mutex.RUnlock()
- r.update(mq, off, true)
- return off
- default:
- }
-
- return -1
-}
-
-func (r *remoteBrokerOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
- localOffset, exist := r.OffsetTable[*mq]
- if !exist {
- r.OffsetTable[*mq] = offset
- return
- }
- if increaseOnly {
- if localOffset < offset {
- r.OffsetTable[*mq] = offset
- }
- } else {
- r.OffsetTable[*mq] = offset
- }
-}
-
-func (r *remoteBrokerOffsetStore) fetchConsumeOffsetFromBroker(group string, mq *primitive.MessageQueue) (int64, error) {
- broker := r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- if broker == "" {
- r.namesrv.UpdateTopicRouteInfo(mq.Topic)
- broker = r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- }
- if broker == "" {
- return int64(-1), fmt.Errorf("broker: %s address not found", mq.BrokerName)
- }
- queryOffsetRequest := &internal.QueryConsumerOffsetRequestHeader{
- ConsumerGroup: group,
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- }
- cmd := remote.NewRemotingCommand(internal.ReqQueryConsumerOffset, queryOffsetRequest, nil)
- res, err := r.client.InvokeSync(context.Background(), broker, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
- if res.Code != internal.ResSuccess {
- return -2, fmt.Errorf("broker response code: %d, remarks: %s", res.Code, res.Remark)
- }
-
- off, err := strconv.ParseInt(res.ExtFields["offset"], 10, 64)
-
- if err != nil {
- return -1, err
- }
-
- return off, nil
-}
-
-func (r *remoteBrokerOffsetStore) updateConsumeOffsetToBroker(group string, mq primitive.MessageQueue, off int64) error {
- broker := r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- if broker == "" {
- r.namesrv.UpdateTopicRouteInfo(mq.Topic)
- broker = r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- }
- if broker == "" {
- return fmt.Errorf("broker: %s address not found", mq.BrokerName)
- }
-
- updateOffsetRequest := &internal.UpdateConsumerOffsetRequestHeader{
- ConsumerGroup: group,
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- CommitOffset: off,
- }
- cmd := remote.NewRemotingCommand(internal.ReqUpdateConsumerOffset, updateOffsetRequest, nil)
- return r.client.InvokeOneWay(context.Background(), broker, cmd, 5*time.Second)
-}
-
-func readFromMemory(table *sync.Map, mq *primitive.MessageQueue) int64 {
- localOffset, exist := table.Load(MessageQueueKey(*mq))
- if !exist {
- return -1
- }
-
- return localOffset.(int64)
-}
diff --git a/consumer/offset_store_test.go b/consumer/offset_store_test.go
deleted file mode 100644
index cfa0eaa..0000000
--- a/consumer/offset_store_test.go
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "path/filepath"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func TestNewLocalFileOffsetStore(t *testing.T) {
- Convey("Given some test cases", t, func() {
- type testCase struct {
- clientId string
- group string
- expectedResult *localFileOffsetStore
- }
- cases := []testCase{
- {
- clientId: "",
- group: "testGroup",
- expectedResult: &localFileOffsetStore{
- group: "testGroup",
- path: filepath.Join(_LocalOffsetStorePath, "/testGroup/offset.json"),
- },
- }, {
- clientId: "192.168.24.1@default",
- group: "",
- expectedResult: &localFileOffsetStore{
- group: "",
- path: filepath.Join(_LocalOffsetStorePath, "/192.168.24.1@default/offset.json"),
- },
- }, {
- clientId: "192.168.24.1@default",
- group: "testGroup",
- expectedResult: &localFileOffsetStore{
- group: "testGroup",
- path: filepath.Join(_LocalOffsetStorePath, "/192.168.24.1@default/testGroup/offset.json"),
- },
- },
- }
-
- for _, value := range cases {
- result := NewLocalFileOffsetStore(value.clientId, value.group).(*localFileOffsetStore)
- value.expectedResult.OffsetTable = result.OffsetTable
- So(result, ShouldResemble, value.expectedResult)
- }
- })
-}
-
-func TestLocalFileOffsetStore(t *testing.T) {
- Convey("Given a local store with a starting value", t, func() {
- localStore := NewLocalFileOffsetStore("192.168.24.1@default", "testGroup")
-
- type offsetCase struct {
- queue *primitive.MessageQueue
- setOffset int64
- expectedOffset int64
- }
- mq := &primitive.MessageQueue{
- Topic: "testTopic",
- BrokerName: "default",
- QueueId: 1,
- }
-
- Convey("test update", func() {
- Convey("when increaseOnly is false", func() {
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 1,
- },
- }
- for _, value := range cases {
- localStore.update(value.queue, value.setOffset, false)
- offset := localStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
-
- Convey("when increaseOnly is true", func() {
- localStore.update(mq, 0, false)
-
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 3,
- },
- }
- for _, value := range cases {
- localStore.update(value.queue, value.setOffset, true)
- offset := localStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
- })
-
- Convey("test persist", func() {
- localStore.update(mq, 1, false)
- offset := localStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, 1)
-
- queues := []*primitive.MessageQueue{mq}
- localStore.persist(queues)
- offset = localStore.read(mq, _ReadFromStore)
- So(offset, ShouldEqual, 1)
-
- localStore.(*localFileOffsetStore).OffsetTable.Delete(MessageQueueKey(*mq))
- offset = localStore.read(mq, _ReadMemoryThenStore)
- So(offset, ShouldEqual, 1)
- })
- })
-}
-
-func TestRemoteBrokerOffsetStore(t *testing.T) {
- Convey("Given a remote store with a starting value", t, func() {
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- namesrv := internal.NewMockNamesrvs(ctrl)
-
- rmqClient := internal.NewMockRMQClient(ctrl)
- remoteStore := NewRemoteOffsetStore("testGroup", rmqClient, namesrv)
-
- type offsetCase struct {
- queue *primitive.MessageQueue
- setOffset int64
- expectedOffset int64
- }
- mq := &primitive.MessageQueue{
- Topic: "testTopic",
- BrokerName: "default",
- QueueId: 1,
- }
-
- Convey("test update", func() {
- Convey("when increaseOnly is false", func() {
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 1,
- },
- }
- for _, value := range cases {
- remoteStore.update(value.queue, value.setOffset, false)
- offset := remoteStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
-
- Convey("when increaseOnly is true", func() {
- remoteStore.update(mq, 0, false)
-
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 3,
- },
- }
- for _, value := range cases {
- remoteStore.update(value.queue, value.setOffset, true)
- offset := remoteStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
- })
-
- Convey("test persist", func() {
- queues := []*primitive.MessageQueue{mq}
-
- namesrv.EXPECT().FindBrokerAddrByName(gomock.Any()).Return("192.168.24.1:10911").MaxTimes(2)
-
- ret := &remote.RemotingCommand{
- Code: internal.ResSuccess,
- ExtFields: map[string]string{
- "offset": "1",
- },
- }
- rmqClient.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(ret, nil).MaxTimes(2)
-
- remoteStore.persist(queues)
- offset := remoteStore.read(mq, _ReadFromStore)
- So(offset, ShouldEqual, 1)
-
- remoteStore.remove(mq)
- offset = remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, -1)
- offset = remoteStore.read(mq, _ReadMemoryThenStore)
- So(offset, ShouldEqual, 1)
-
- })
-
- Convey("test remove", func() {
- remoteStore.update(mq, 1, false)
- offset := remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, 1)
-
- remoteStore.remove(mq)
- offset = remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, -1)
- })
- })
-}
diff --git a/consumer/option.go b/consumer/option.go
deleted file mode 100644
index 330f6c4..0000000
--- a/consumer/option.go
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-type consumerOptions struct {
- internal.ClientOptions
-
- /**
- * Backtracking consumption time with second precision. Time format is
- * 20131223171201<br>
- * Implying Seventeen twelve and 01 seconds on December 23, 2013 year<br>
- * Default backtracking consumption time Half an hour ago.
- */
- ConsumeTimestamp string
-
- // The socket timeout in milliseconds
- ConsumerPullTimeout time.Duration
-
- // Concurrently max span offset.it has no effect on sequential consumption
- ConsumeConcurrentlyMaxSpan int
-
- // Flow control threshold on queue level, each message queue will cache at most 1000 messages by default,
- // Consider the {PullBatchSize}, the instantaneous value may exceed the limit
- PullThresholdForQueue int64
-
- // Limit the cached message size on queue level, each message queue will cache at most 100 MiB messages by default,
- // Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit
- //
- // The size of a message only measured by message body, so it's not accurate
- PullThresholdSizeForQueue int
-
- // Flow control threshold on topic level, default value is -1(Unlimited)
- //
- // The value of {@code pullThresholdForQueue} will be overwrote and calculated based on
- // {@code pullThresholdForTopic} if it is't unlimited
- //
- // For example, if the value of pullThresholdForTopic is 1000 and 10 message queues are assigned to this consumer,
- // then pullThresholdForQueue will be set to 100
- PullThresholdForTopic int
-
- // Limit the cached message size on topic level, default value is -1 MiB(Unlimited)
- //
- // The value of {@code pullThresholdSizeForQueue} will be overwrote and calculated based on
- // {@code pullThresholdSizeForTopic} if it is't unlimited
- //
- // For example, if the value of pullThresholdSizeForTopic is 1000 MiB and 10 message queues are
- // assigned to this consumer, then pullThresholdSizeForQueue will be set to 100 MiB
- PullThresholdSizeForTopic int
-
- // Message pull Interval
- PullInterval time.Duration
-
- // Batch consumption size
- ConsumeMessageBatchMaxSize int
-
- // Batch pull size
- PullBatchSize int32
-
- // Whether update subscription relationship when every pull
- PostSubscriptionWhenPull bool
-
- // Max re-consume times. -1 means 16 times.
- //
- // If messages are re-consumed more than {@link #maxReconsumeTimes} before Success, it's be directed to a deletion
- // queue waiting.
- MaxReconsumeTimes int32
-
- // Suspending pulling time for cases requiring slow pulling like flow-control scenario.
- SuspendCurrentQueueTimeMillis time.Duration
-
- // Maximum amount of time a message may block the consuming thread.
- ConsumeTimeout time.Duration
-
- ConsumerModel MessageModel
- Strategy AllocateStrategy
- ConsumeOrderly bool
- FromWhere ConsumeFromWhere
-
- Interceptors []primitive.Interceptor
- // TODO traceDispatcher
- MaxTimeConsumeContinuously time.Duration
- //
- AutoCommit bool
- RebalanceLockInterval time.Duration
-
- Resolver primitive.NsResolver
-}
-
-func defaultPushConsumerOptions() consumerOptions {
- opts := consumerOptions{
- ClientOptions: internal.DefaultClientOptions(),
- Strategy: AllocateByAveragely,
- MaxTimeConsumeContinuously: time.Duration(60 * time.Second),
- RebalanceLockInterval: 20 * time.Second,
- MaxReconsumeTimes: -1,
- ConsumerModel: Clustering,
- AutoCommit: true,
- Resolver: primitive.NewHttpResolver("DEFAULT"),
- }
- opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
- return opts
-}
-
-type Option func(*consumerOptions)
-
-func defaultPullConsumerOptions() consumerOptions {
- opts := consumerOptions{
- ClientOptions: internal.DefaultClientOptions(),
- Resolver: primitive.NewHttpResolver("DEFAULT"),
- }
- opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
- return opts
-}
-
-func WithConsumerModel(m MessageModel) Option {
- return func(options *consumerOptions) {
- options.ConsumerModel = m
- }
-}
-
-func WithConsumeFromWhere(w ConsumeFromWhere) Option {
- return func(options *consumerOptions) {
- options.FromWhere = w
- }
-}
-
-func WithConsumerOrder(order bool) Option {
- return func(options *consumerOptions) {
- options.ConsumeOrderly = order
- }
-}
-
-func WithConsumeMessageBatchMaxSize(consumeMessageBatchMaxSize int) Option {
- return func(options *consumerOptions) {
- options.ConsumeMessageBatchMaxSize = consumeMessageBatchMaxSize
- }
-}
-
-// WithChainConsumerInterceptor returns a ConsumerOption that specifies the chained interceptor for consumer.
-// The first interceptor will be the outer most, while the last interceptor will be the inner most wrapper
-// around the real call.
-func WithInterceptor(fs ...primitive.Interceptor) Option {
- return func(options *consumerOptions) {
- options.Interceptors = append(options.Interceptors, fs...)
- }
-}
-
-// WithGroupName set group name address
-func WithGroupName(group string) Option {
- return func(opts *consumerOptions) {
- if group == "" {
- return
- }
- opts.GroupName = group
- }
-}
-
-func WithInstance(name string) Option {
- return func(options *consumerOptions) {
- options.InstanceName = name
- }
-}
-
-// WithNamespace set the namespace of consumer
-func WithNamespace(namespace string) Option {
- return func(opts *consumerOptions) {
- opts.Namespace = namespace
- }
-}
-
-func WithVIPChannel(enable bool) Option {
- return func(opts *consumerOptions) {
- opts.VIPChannelEnabled = enable
- }
-}
-
-// WithRetry return a Option that specifies the retry times when send failed.
-// TODO: use retry middleware instead
-func WithRetry(retries int) Option {
- return func(opts *consumerOptions) {
- opts.RetryTimes = retries
- }
-}
-
-func WithCredentials(c primitive.Credentials) Option {
- return func(options *consumerOptions) {
- options.ClientOptions.Credentials = c
- }
-}
-
-// WithMaxReconsumeTimes set MaxReconsumeTimes of options, if message reconsume greater than MaxReconsumeTimes, it will
-// be sent to retry or dlq topic. more info reference by examples/consumer/retry.
-func WithMaxReconsumeTimes(times int32) Option {
- return func(opts *consumerOptions) {
- opts.MaxReconsumeTimes = times
- }
-}
-
-func WithStrategy(strategy AllocateStrategy) Option {
- return func(opts *consumerOptions) {
- opts.Strategy = strategy
- }
-}
-
-func WithPullBatchSize(batchSize int32) Option {
- return func(options *consumerOptions) {
- options.PullBatchSize = batchSize
- }
-}
-
-func WithRebalanceLockInterval(interval time.Duration) Option {
- return func(options *consumerOptions) {
- options.RebalanceLockInterval = interval
- }
-}
-
-func WithAutoCommit(auto bool) Option {
- return func(options *consumerOptions) {
- options.AutoCommit = auto
- }
-}
-
-func WithSuspendCurrentQueueTimeMillis(suspendT time.Duration) Option {
- return func(options *consumerOptions) {
- options.SuspendCurrentQueueTimeMillis = suspendT
- }
-}
-
-func WithPullInterval(interval time.Duration) Option {
- return func(options *consumerOptions) {
- options.PullInterval = interval
- }
-}
-
-// WithNsResolver set nameserver resolver to fetch nameserver addr
-func WithNsResolver(resolver primitive.NsResolver) Option {
- return func(options *consumerOptions) {
- options.Resolver = resolver
- }
-}
-
-// WithNameServer set NameServer address, only support one NameServer cluster in alpha2
-func WithNameServer(nameServers primitive.NamesrvAddr) Option {
- return func(options *consumerOptions) {
- options.Resolver = primitive.NewPassthroughResolver(nameServers)
- }
-}
-
-// WithNameServerDomain set NameServer domain
-func WithNameServerDomain(nameServerUrl string) Option {
- return func(opts *consumerOptions) {
- opts.Resolver = primitive.NewHttpResolver("DEFAULT", nameServerUrl)
- }
-}
diff --git a/consumer/process_queue.go b/consumer/process_queue.go
deleted file mode 100644
index 76a9236..0000000
--- a/consumer/process_queue.go
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "strconv"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/emirpasic/gods/maps/treemap"
- "github.com/emirpasic/gods/utils"
- gods_util "github.com/emirpasic/gods/utils"
- uatomic "go.uber.org/atomic"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- _RebalanceLockMaxTime = 30 * time.Second
- _RebalanceInterval = 20 * time.Second
- _PullMaxIdleTime = 120 * time.Second
-)
-
-type processQueue struct {
- cachedMsgCount int64
- cachedMsgSize int64
- tryUnlockTimes int64
- queueOffsetMax int64
- msgAccCnt int64
- msgCache *treemap.Map
- mutex sync.RWMutex
- consumeLock sync.Mutex
- consumingMsgOrderlyTreeMap *treemap.Map
- dropped *uatomic.Bool
- lastPullTime atomic.Value
- lastConsumeTime atomic.Value
- locked *uatomic.Bool
- lastLockTime atomic.Value
- consuming bool
- lockConsume sync.Mutex
- msgCh chan []*primitive.MessageExt
- order bool
-}
-
-func newProcessQueue(order bool) *processQueue {
- consumingMsgOrderlyTreeMap := treemap.NewWith(gods_util.Int64Comparator)
-
- lastConsumeTime := atomic.Value{}
- lastConsumeTime.Store(time.Now())
-
- lastLockTime := atomic.Value{}
- lastLockTime.Store(time.Now())
-
- lastPullTime := atomic.Value{}
- lastPullTime.Store(time.Now())
-
- pq := &processQueue{
- msgCache: treemap.NewWith(utils.Int64Comparator),
- lastPullTime: lastPullTime,
- lastConsumeTime: lastConsumeTime,
- lastLockTime: lastLockTime,
- msgCh: make(chan []*primitive.MessageExt, 32),
- consumingMsgOrderlyTreeMap: consumingMsgOrderlyTreeMap,
- order: order,
- locked: uatomic.NewBool(false),
- dropped: uatomic.NewBool(false),
- }
- return pq
-}
-
-func (pq *processQueue) putMessage(messages ...*primitive.MessageExt) {
- if len(messages) == 0 {
- return
- }
- pq.mutex.Lock()
- if pq.IsDroppd() {
- pq.mutex.Unlock()
- return
- }
- if !pq.order {
- pq.msgCh <- messages
- }
- validMessageCount := 0
- for idx := range messages {
- msg := messages[idx]
- _, found := pq.msgCache.Get(msg.QueueOffset)
- if found {
- continue
- }
- _, found = pq.consumingMsgOrderlyTreeMap.Get(msg.QueueOffset)
- if found {
- continue
- }
- pq.msgCache.Put(msg.QueueOffset, msg)
- validMessageCount++
- pq.queueOffsetMax = msg.QueueOffset
- atomic.AddInt64(&pq.cachedMsgSize, int64(len(msg.Body)))
- }
- pq.mutex.Unlock()
-
- atomic.AddInt64(&pq.cachedMsgCount, int64(validMessageCount))
-
- if pq.msgCache.Size() > 0 && !pq.consuming {
- pq.consuming = true
- }
-
- msg := messages[len(messages)-1]
- maxOffset, err := strconv.ParseInt(msg.GetProperty(primitive.PropertyMaxOffset), 10, 64)
- if err != nil {
- acc := maxOffset - msg.QueueOffset
- if acc > 0 {
- pq.msgAccCnt = acc
- }
- }
-}
-
-func (pq *processQueue) WithLock(lock bool) {
- pq.locked.Store(lock)
-}
-
-func (pq *processQueue) IsLock() bool {
- return pq.locked.Load()
-}
-
-func (pq *processQueue) WithDropped(dropped bool) {
- pq.dropped.Store(dropped)
-}
-
-func (pq *processQueue) IsDroppd() bool {
- return pq.dropped.Load()
-}
-
-func (pq *processQueue) UpdateLastConsumeTime() {
- pq.lastConsumeTime.Store(time.Now())
-}
-
-func (pq *processQueue) LastConsumeTime() time.Time {
- return pq.lastConsumeTime.Load().(time.Time)
-}
-
-func (pq *processQueue) UpdateLastLockTime() {
- pq.lastLockTime.Store(time.Now())
-}
-
-func (pq *processQueue) LastLockTime() time.Time {
- return pq.lastLockTime.Load().(time.Time)
-}
-
-func (pq *processQueue) LastPullTime() time.Time {
- return pq.lastPullTime.Load().(time.Time)
-}
-
-func (pq *processQueue) UpdateLastPullTime() {
- pq.lastPullTime.Store(time.Now())
-}
-
-func (pq *processQueue) makeMessageToCosumeAgain(messages ...*primitive.MessageExt) {
- pq.mutex.Lock()
- for _, msg := range messages {
- pq.consumingMsgOrderlyTreeMap.Remove(msg.QueueOffset)
- pq.msgCache.Put(msg.QueueOffset, msg)
- }
-
- pq.mutex.Unlock()
-}
-
-func (pq *processQueue) removeMessage(messages ...*primitive.MessageExt) int64 {
- result := int64(-1)
- pq.mutex.Lock()
- pq.UpdateLastConsumeTime()
- if !pq.msgCache.Empty() {
- result = pq.queueOffsetMax + 1
- removedCount := 0
- for idx := range messages {
- msg := messages[idx]
- _, found := pq.msgCache.Get(msg.QueueOffset)
- if !found {
- continue
- }
- pq.msgCache.Remove(msg.QueueOffset)
- removedCount++
- atomic.AddInt64(&pq.cachedMsgSize, int64(-len(msg.Body)))
- }
- atomic.AddInt64(&pq.cachedMsgCount, int64(-removedCount))
- }
- if !pq.msgCache.Empty() {
- first, _ := pq.msgCache.Min()
- result = first.(int64)
- }
- pq.mutex.Unlock()
- return result
-}
-
-func (pq *processQueue) isLockExpired() bool {
- return time.Now().Sub(pq.LastLockTime()) > _RebalanceLockMaxTime
-}
-
-func (pq *processQueue) isPullExpired() bool {
- return time.Now().Sub(pq.LastPullTime()) > _PullMaxIdleTime
-}
-
-func (pq *processQueue) cleanExpiredMsg(consumer defaultConsumer) {
- if consumer.option.ConsumeOrderly {
- return
- }
- var loop = 16
- if pq.msgCache.Size() < 16 {
- loop = pq.msgCache.Size()
- }
-
- for i := 0; i < loop; i++ {
- pq.mutex.RLock()
- if pq.msgCache.Empty() {
- pq.mutex.RLock()
- return
- }
- _, firstValue := pq.msgCache.Min()
- msg := firstValue.(*primitive.MessageExt)
- startTime := msg.GetProperty(primitive.PropertyConsumeStartTime)
- if startTime != "" {
- st, err := strconv.ParseInt(startTime, 10, 64)
- if err != nil {
- rlog.Warning("parse message start consume time error", map[string]interface{}{
- "time": startTime,
- rlog.LogKeyUnderlayError: err,
- })
- continue
- }
- if time.Now().Unix()-st <= int64(consumer.option.ConsumeTimeout) {
- pq.mutex.RLock()
- return
- }
- }
- pq.mutex.RLock()
-
- err := consumer.sendBack(msg, 3)
- if err != nil {
- rlog.Error("send message back to broker error when clean expired messages", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- continue
- }
- pq.removeMessage(msg)
- }
-}
-
-func (pq *processQueue) getMaxSpan() int {
- pq.mutex.RLock()
- defer pq.mutex.RUnlock()
- if pq.msgCache.Size() == 0 {
- return 0
- }
- firstKey, _ := pq.msgCache.Min()
- lastKey, _ := pq.msgCache.Max()
- return int(lastKey.(int64) - firstKey.(int64))
-}
-
-func (pq *processQueue) getMessages() []*primitive.MessageExt {
- return <-pq.msgCh
-}
-
-func (pq *processQueue) takeMessages(number int) []*primitive.MessageExt {
- for pq.msgCache.Empty() {
- time.Sleep(10 * time.Millisecond)
- }
- result := make([]*primitive.MessageExt, number)
- i := 0
- pq.mutex.Lock()
- for ; i < number; i++ {
- k, v := pq.msgCache.Min()
- if v == nil {
- break
- }
- result[i] = v.(*primitive.MessageExt)
- pq.consumingMsgOrderlyTreeMap.Put(k, v)
- pq.msgCache.Remove(k)
- }
- pq.mutex.Unlock()
- return result[:i]
-}
-
-func (pq *processQueue) Min() int64 {
- if pq.msgCache.Empty() {
- return -1
- }
- k, _ := pq.msgCache.Min()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) Max() int64 {
- if pq.msgCache.Empty() {
- return -1
- }
- k, _ := pq.msgCache.Max()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) MinOrderlyCache() int64 {
- if pq.consumingMsgOrderlyTreeMap.Empty() {
- return -1
- }
- k, _ := pq.consumingMsgOrderlyTreeMap.Min()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) MaxOrderlyCache() int64 {
- if pq.consumingMsgOrderlyTreeMap.Empty() {
- return -1
- }
- k, _ := pq.consumingMsgOrderlyTreeMap.Max()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) clear() {
- pq.mutex.Lock()
- defer pq.mutex.Unlock()
- pq.msgCache.Clear()
- pq.cachedMsgCount = 0
- pq.cachedMsgSize = 0
- pq.queueOffsetMax = 0
-}
-
-func (pq *processQueue) commit() int64 {
- pq.mutex.Lock()
- defer pq.mutex.Unlock()
-
- var offset int64
- iter, _ := pq.consumingMsgOrderlyTreeMap.Max()
- if iter != nil {
- offset = iter.(int64)
- }
- pq.cachedMsgCount -= int64(pq.consumingMsgOrderlyTreeMap.Size())
- pq.consumingMsgOrderlyTreeMap.Each(func(key interface{}, value interface{}) {
- msg := value.(*primitive.MessageExt)
- pq.cachedMsgSize -= int64(len(msg.Body))
- })
- pq.consumingMsgOrderlyTreeMap.Clear()
- return offset + 1
-}
-
-func (pq *processQueue) currentInfo() internal.ProcessQueueInfo {
- pq.mutex.RLock()
- defer pq.mutex.RUnlock()
- info := internal.ProcessQueueInfo{
- Locked: pq.locked.Load(),
- TryUnlockTimes: pq.tryUnlockTimes,
- LastLockTimestamp: pq.LastLockTime().UnixNano() / int64(time.Millisecond),
- Dropped: pq.dropped.Load(),
- LastPullTimestamp: pq.LastPullTime().UnixNano() / int64(time.Millisecond),
- LastConsumeTimestamp: pq.LastConsumeTime().UnixNano() / int64(time.Millisecond),
- }
-
- if !pq.msgCache.Empty() {
- info.CachedMsgMinOffset = pq.Min()
- info.CachedMsgMaxOffset = pq.Max()
- info.CachedMsgCount = pq.msgCache.Size()
- info.CachedMsgSizeInMiB = pq.cachedMsgSize / int64(1024*1024)
- }
-
- if !pq.consumingMsgOrderlyTreeMap.Empty() {
- info.TransactionMsgMinOffset = pq.MinOrderlyCache()
- info.TransactionMsgMaxOffset = pq.MaxOrderlyCache()
- info.TransactionMsgCount = pq.consumingMsgOrderlyTreeMap.Size()
- }
-
- return info
-}
diff --git a/consumer/pull_consumer.go b/consumer/pull_consumer.go
deleted file mode 100644
index 874973b..0000000
--- a/consumer/pull_consumer.go
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- errors2 "github.com/apache/rocketmq-client-go/v2/errors"
- "sync"
- "sync/atomic"
-
- "github.com/pkg/errors"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type PullConsumer interface {
- // Start
- Start()
-
- // Shutdown refuse all new pull operation, finish all submitted.
- Shutdown()
-
- // Pull pull message of topic, selector indicate which queue to pull.
- Pull(ctx context.Context, topic string, selector MessageSelector, numbers int) (*primitive.PullResult, error)
-
- // PullFrom pull messages of queue from the offset to offset + numbers
- PullFrom(ctx context.Context, queue *primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error)
-
- // updateOffset update offset of queue in mem
- UpdateOffset(queue *primitive.MessageQueue, offset int64) error
-
- // PersistOffset persist all offset in mem.
- PersistOffset(ctx context.Context) error
-
- // CurrentOffset return the current offset of queue in mem.
- CurrentOffset(queue *primitive.MessageQueue) (int64, error)
-}
-
-var (
- queueCounterTable sync.Map
-)
-
-type defaultPullConsumer struct {
- *defaultConsumer
-
- option consumerOptions
- client internal.RMQClient
- GroupName string
- Model MessageModel
- UnitMode bool
-
- interceptor primitive.Interceptor
-}
-
-func NewPullConsumer(options ...Option) (*defaultPullConsumer, error) {
- defaultOpts := defaultPullConsumerOptions()
- for _, apply := range options {
- apply(&defaultOpts)
- }
-
- srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
- if err != nil {
- return nil, errors.Wrap(err, "new Namesrv failed.")
- }
-
- defaultOpts.Namesrv = srvs
- dc := &defaultConsumer{
- client: internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil),
- consumerGroup: defaultOpts.GroupName,
- cType: _PullConsume,
- state: int32(internal.StateCreateJust),
- prCh: make(chan PullRequest, 4),
- model: defaultOpts.ConsumerModel,
- option: defaultOpts,
- }
- if dc.client == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = dc.client.GetNameSrv()
-
- c := &defaultPullConsumer{
- defaultConsumer: dc,
- }
- return c, nil
-}
-
-func (c *defaultPullConsumer) Start() error {
- atomic.StoreInt32(&c.state, int32(internal.StateRunning))
-
- var err error
- c.once.Do(func() {
- err = c.start()
- if err != nil {
- return
- }
- })
-
- return err
-}
-
-func (c *defaultPullConsumer) Pull(ctx context.Context, topic string, selector MessageSelector, numbers int) (*primitive.PullResult, error) {
- mq := c.getNextQueueOf(topic)
- if mq == nil {
- return nil, fmt.Errorf("prepard to pull topic: %s, but no queue is founded", topic)
- }
-
- data := buildSubscriptionData(mq.Topic, selector)
- result, err := c.pull(context.Background(), mq, data, c.nextOffsetOf(mq), numbers)
-
- if err != nil {
- return nil, err
- }
-
- c.processPullResult(mq, result, data)
- return result, nil
-}
-
-func (c *defaultPullConsumer) getNextQueueOf(topic string) *primitive.MessageQueue {
- queues, err := c.defaultConsumer.client.GetNameSrv().FetchSubscribeMessageQueues(topic)
- if err != nil && len(queues) > 0 {
- rlog.Error("get next mq error", map[string]interface{}{
- rlog.LogKeyTopic: topic,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return nil
- }
- var index int64
- v, exist := queueCounterTable.Load(topic)
- if !exist {
- index = -1
- queueCounterTable.Store(topic, int64(0))
- } else {
- index = v.(int64)
- }
-
- return queues[int(atomic.AddInt64(&index, 1))%len(queues)]
-}
-
-// SubscribeWithChan ack manually
-func (c *defaultPullConsumer) SubscribeWithChan(topic, selector MessageSelector) (chan *primitive.Message, error) {
- return nil, nil
-}
-
-// SubscribeWithFunc ack automatic
-func (c *defaultPullConsumer) SubscribeWithFunc(topic, selector MessageSelector,
- f func(msg *primitive.Message) ConsumeResult) error {
- return nil
-}
-
-func (c *defaultPullConsumer) ACK(msg *primitive.Message, result ConsumeResult) {
-
-}
-
-func (dc *defaultConsumer) checkPull(ctx context.Context, mq *primitive.MessageQueue, offset int64, numbers int) error {
- err := dc.makeSureStateOK()
- if err != nil {
- return err
- }
-
- if mq == nil {
- return errors2.ErrMQEmpty
- }
-
- if offset < 0 {
- return errors2.ErrOffset
- }
-
- if numbers <= 0 {
- return errors2.ErrNumbers
- }
- return nil
-}
-
-// TODO: add timeout limit
-// TODO: add hook
-func (c *defaultPullConsumer) pull(ctx context.Context, mq *primitive.MessageQueue, data *internal.SubscriptionData,
- offset int64, numbers int) (*primitive.PullResult, error) {
-
- if err := c.checkPull(ctx, mq, offset, numbers); err != nil {
- return nil, err
- }
-
- c.subscriptionAutomatically(mq.Topic)
-
- sysFlag := buildSysFlag(false, true, true, false)
-
- pullResp, err := c.pullInner(ctx, mq, data, offset, numbers, sysFlag, 0)
- if err != nil {
- return nil, err
- }
- c.processPullResult(mq, pullResp, data)
-
- return pullResp, err
-}
-
-func (c *defaultPullConsumer) makeSureStateOK() error {
- if atomic.LoadInt32(&c.state) != int32(internal.StateRunning) {
- return fmt.Errorf("the consumer state is [%d], not running", c.state)
- }
- return nil
-}
-
-func (c *defaultPullConsumer) nextOffsetOf(queue *primitive.MessageQueue) int64 {
- return c.computePullFromWhere(queue)
-}
-
-// PullFrom pull messages of queue from the offset to offset + numbers
-func (c *defaultPullConsumer) PullFrom(ctx context.Context, queue *primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error) {
- if err := c.checkPull(ctx, queue, offset, numbers); err != nil {
- return nil, err
- }
-
- selector := MessageSelector{}
- data := buildSubscriptionData(queue.Topic, selector)
-
- return c.pull(ctx, queue, data, offset, numbers)
-}
-
-// updateOffset update offset of queue in mem
-func (c *defaultPullConsumer) UpdateOffset(queue *primitive.MessageQueue, offset int64) error {
- return c.updateOffset(queue, offset)
-}
-
-// PersistOffset persist all offset in mem.
-func (c *defaultPullConsumer) PersistOffset(ctx context.Context) error {
- return c.persistConsumerOffset()
-}
-
-// CurrentOffset return the current offset of queue in mem.
-func (c *defaultPullConsumer) CurrentOffset(queue *primitive.MessageQueue) (int64, error) {
- v := c.queryOffset(queue)
- return v, nil
-}
-
-// Shutdown close defaultConsumer, refuse new request.
-func (c *defaultPullConsumer) Shutdown() error {
- return c.defaultConsumer.shutdown()
-}
diff --git a/consumer/pull_consumer_test.go b/consumer/pull_consumer_test.go
deleted file mode 100644
index f7bc454..0000000
--- a/consumer/pull_consumer_test.go
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
diff --git a/consumer/push_consumer.go b/consumer/push_consumer.go
deleted file mode 100644
index 8642aa4..0000000
--- a/consumer/push_consumer.go
+++ /dev/null
@@ -1,1254 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- errors2 "github.com/apache/rocketmq-client-go/v2/errors"
- "math"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/pkg/errors"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-// In most scenarios, this is the mostly recommended usage to consume messages.
-//
-// Technically speaking, this push client is virtually a wrapper of the underlying pull service. Specifically, on
-// arrival of messages pulled from brokers, it roughly invokes the registered callback handler to feed the messages.
-//
-// See quick start/Consumer in the example module for a typical usage.
-//
-// <strong>Thread Safety:</strong> After initialization, the instance can be regarded as thread-safe.
-
-const (
- Mb = 1024 * 1024
-)
-
-type PushConsumerCallback struct {
- topic string
- f func(context.Context, ...*primitive.MessageExt) (ConsumeResult, error)
-}
-
-func (callback PushConsumerCallback) UniqueID() string {
- return callback.topic
-}
-
-type pushConsumer struct {
- *defaultConsumer
- queueFlowControlTimes int
- queueMaxSpanFlowControlTimes int
- consumeFunc utils.Set
- submitToConsume func(*processQueue, *primitive.MessageQueue)
- subscribedTopic map[string]string
- interceptor primitive.Interceptor
- queueLock *QueueLock
- done chan struct{}
- closeOnce sync.Once
-}
-
-func NewPushConsumer(opts ...Option) (*pushConsumer, error) {
- defaultOpts := defaultPushConsumerOptions()
- for _, apply := range opts {
- apply(&defaultOpts)
- }
- srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
- if err != nil {
- return nil, errors.Wrap(err, "new Namesrv failed.")
- }
- if !defaultOpts.Credentials.IsEmpty() {
- srvs.SetCredentials(defaultOpts.Credentials)
- }
- defaultOpts.Namesrv = srvs
-
- if defaultOpts.Namespace != "" {
- defaultOpts.GroupName = defaultOpts.Namespace + "%" + defaultOpts.GroupName
- }
-
- dc := &defaultConsumer{
- client: internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil),
- consumerGroup: defaultOpts.GroupName,
- cType: _PushConsume,
- state: int32(internal.StateCreateJust),
- prCh: make(chan PullRequest, 4),
- model: defaultOpts.ConsumerModel,
- consumeOrderly: defaultOpts.ConsumeOrderly,
- fromWhere: defaultOpts.FromWhere,
- allocate: defaultOpts.Strategy,
- option: defaultOpts,
- }
- if dc.client == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = dc.client.GetNameSrv()
-
- p := &pushConsumer{
- defaultConsumer: dc,
- subscribedTopic: make(map[string]string, 0),
- queueLock: newQueueLock(),
- done: make(chan struct{}, 1),
- consumeFunc: utils.NewSet(),
- }
- dc.mqChanged = p.messageQueueChanged
- if p.consumeOrderly {
- p.submitToConsume = p.consumeMessageOrderly
- } else {
- p.submitToConsume = p.consumeMessageCurrently
- }
-
- p.interceptor = primitive.ChainInterceptors(p.option.Interceptors...)
-
- return p, nil
-}
-
-func (pc *pushConsumer) Start() error {
- var err error
- pc.once.Do(func() {
- rlog.Info("the consumer start beginning", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- "messageModel": pc.model,
- "unitMode": pc.unitMode,
- })
- atomic.StoreInt32(&pc.state, int32(internal.StateStartFailed))
- pc.validate()
-
- err = pc.client.RegisterConsumer(pc.consumerGroup, pc)
- if err != nil {
- rlog.Error("the consumer group has been created, specify another one", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- err = errors2.ErrCreated
- return
- }
-
- err = pc.defaultConsumer.start()
- if err != nil {
- return
- }
-
- go func() {
- // todo start clean msg expired
- for {
- select {
- case pr := <-pc.prCh:
- go func() {
- pc.pullMessage(&pr)
- }()
- case <-pc.done:
- rlog.Info("push consumer close pullConsumer listener.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- }()
-
- go primitive.WithRecover(func() {
- // initial lock.
- if !pc.consumeOrderly {
- return
- }
-
- time.Sleep(1000 * time.Millisecond)
- pc.lockAll()
-
- lockTicker := time.NewTicker(pc.option.RebalanceLockInterval)
- defer lockTicker.Stop()
- for {
- select {
- case <-lockTicker.C:
- pc.lockAll()
- case <-pc.done:
- rlog.Info("push consumer close tick.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- })
- })
-
- if err != nil {
- return err
- }
-
- pc.client.UpdateTopicRouteInfo()
- for k := range pc.subscribedTopic {
- _, exist := pc.topicSubscribeInfoTable.Load(k)
- if !exist {
- pc.client.Shutdown()
- return fmt.Errorf("the topic=%s route info not found, it may not exist", k)
- }
- }
- pc.client.CheckClientInBroker()
- pc.client.SendHeartbeatToAllBrokerWithLock()
- pc.client.RebalanceImmediately()
-
- return err
-}
-
-func (pc *pushConsumer) Shutdown() error {
- var err error
- pc.closeOnce.Do(func() {
- close(pc.done)
-
- pc.client.UnregisterConsumer(pc.consumerGroup)
- err = pc.defaultConsumer.shutdown()
- })
-
- return err
-}
-
-func (pc *pushConsumer) Subscribe(topic string, selector MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (ConsumeResult, error)) error {
- if atomic.LoadInt32(&pc.state) == int32(internal.StateStartFailed) ||
- atomic.LoadInt32(&pc.state) == int32(internal.StateShutdown) {
- return errors2.ErrStartTopic
- }
-
- if pc.option.Namespace != "" {
- topic = pc.option.Namespace + "%" + topic
- }
- data := buildSubscriptionData(topic, selector)
- pc.subscriptionDataTable.Store(topic, data)
- pc.subscribedTopic[topic] = ""
-
- pc.consumeFunc.Add(&PushConsumerCallback{
- f: f,
- topic: topic,
- })
- return nil
-}
-
-func (pc *pushConsumer) Unsubscribe(topic string) error {
- if pc.option.Namespace != "" {
- topic = pc.option.Namespace + "%" + topic
- }
- pc.subscriptionDataTable.Delete(topic)
- return nil
-}
-
-func (pc *pushConsumer) Rebalance() {
- pc.defaultConsumer.doBalance()
-}
-
-func (pc *pushConsumer) PersistConsumerOffset() error {
- return pc.defaultConsumer.persistConsumerOffset()
-}
-
-func (pc *pushConsumer) UpdateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
- pc.defaultConsumer.updateTopicSubscribeInfo(topic, mqs)
-}
-
-func (pc *pushConsumer) IsSubscribeTopicNeedUpdate(topic string) bool {
- return pc.defaultConsumer.isSubscribeTopicNeedUpdate(topic)
-}
-
-func (pc *pushConsumer) SubscriptionDataList() []*internal.SubscriptionData {
- return pc.defaultConsumer.SubscriptionDataList()
-}
-
-func (pc *pushConsumer) IsUnitMode() bool {
- return pc.unitMode
-}
-
-func (pc *pushConsumer) GetcType() string {
- return string(pc.cType)
-}
-
-func (pc *pushConsumer) GetModel() string {
- return pc.model.String()
-}
-
-func (pc *pushConsumer) GetWhere() string {
- switch pc.fromWhere {
- case ConsumeFromLastOffset:
- return "CONSUME_FROM_LAST_OFFSET"
- case ConsumeFromFirstOffset:
- return "CONSUME_FROM_FIRST_OFFSET"
- case ConsumeFromTimestamp:
- return "CONSUME_FROM_TIMESTAMP"
- default:
- return "UNKOWN"
- }
-
-}
-
-func (pc *pushConsumer) ConsumeMessageDirectly(msg *primitive.MessageExt, brokerName string) *internal.ConsumeMessageDirectlyResult {
- var msgs = []*primitive.MessageExt{msg}
- var mq = &primitive.MessageQueue{
- Topic: msg.Topic,
- BrokerName: brokerName,
- QueueId: msg.Queue.QueueId,
- }
-
- beginTime := time.Now()
- pc.resetRetryAndNamespace(msgs)
- var result ConsumeResult
-
- var err error
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: msgs,
- }
- ctx := context.Background()
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
- concurrentCtx := primitive.NewConsumeConcurrentlyContext()
- concurrentCtx.MQ = *mq
- ctx = primitive.WithConcurrentlyCtx(ctx, concurrentCtx)
-
- result, err = pc.consumeInner(ctx, msgs)
-
- consumeRT := time.Now().Sub(beginTime)
-
- res := &internal.ConsumeMessageDirectlyResult{
- Order: false,
- AutoCommit: true,
- SpentTimeMills: int64(consumeRT / time.Millisecond),
- }
-
- if err != nil {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.ExceptionReturn)
- res.ConsumeResult = internal.ThrowException
- res.Remark = err.Error()
- } else if result == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- res.ConsumeResult = internal.ConsumeSuccess
- } else if result == ConsumeRetryLater {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- res.ConsumeResult = internal.ConsumeRetryLater
- }
-
- pc.stat.increaseConsumeRT(pc.consumerGroup, mq.Topic, int64(consumeRT/time.Millisecond))
-
- return res
-}
-
-func (pc *pushConsumer) GetConsumerRunningInfo() *internal.ConsumerRunningInfo {
- info := internal.NewConsumerRunningInfo()
-
- pc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- topic := key.(string)
- info.SubscriptionData[value.(*internal.SubscriptionData)] = true
- status := internal.ConsumeStatus{
- PullRT: pc.stat.getPullRT(topic, pc.consumerGroup).avgpt,
- PullTPS: pc.stat.getPullTPS(topic, pc.consumerGroup).tps,
- ConsumeRT: pc.stat.getConsumeRT(topic, pc.consumerGroup).avgpt,
- ConsumeOKTPS: pc.stat.getConsumeOKTPS(topic, pc.consumerGroup).tps,
- ConsumeFailedTPS: pc.stat.getConsumeFailedTPS(topic, pc.consumerGroup).tps,
- ConsumeFailedMsgs: pc.stat.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + pc.consumerGroup).sum,
- }
- info.StatusTable[topic] = status
- return true
- })
-
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- pInfo := pq.currentInfo()
- pInfo.CommitOffset = pc.storage.read(&mq, _ReadMemoryThenStore)
- info.MQTable[mq] = pInfo
- return true
- })
-
- nsAddr := ""
- for _, value := range pc.client.GetNameSrv().AddrList() {
- nsAddr += fmt.Sprintf("%s;", value)
- }
- info.Properties[internal.PropNameServerAddr] = nsAddr
- info.Properties[internal.PropConsumeType] = string(pc.cType)
- info.Properties[internal.PropConsumeOrderly] = strconv.FormatBool(pc.consumeOrderly)
- info.Properties[internal.PropThreadPoolCoreSize] = "-1"
- info.Properties[internal.PropConsumerStartTimestamp] = strconv.FormatInt(pc.consumerStartTimestamp, 10)
- return info
-}
-
-func (pc *pushConsumer) messageQueueChanged(topic string, mqAll, mqDivided []*primitive.MessageQueue) {
- v, exit := pc.subscriptionDataTable.Load(topic)
- if !exit {
- return
- }
- data := v.(*internal.SubscriptionData)
- newVersion := time.Now().UnixNano()
- rlog.Info("the MessageQueue changed, version also updated", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: data.SubVersion,
- rlog.LogKeyValueChangedTo: newVersion,
- })
- data.SubVersion = newVersion
-
- // TODO: optimize
- count := 0
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- count++
- return true
- })
- if count > 0 {
- if pc.option.PullThresholdForTopic != -1 {
- newVal := pc.option.PullThresholdForTopic / count
- if newVal == 0 {
- newVal = 1
- }
- rlog.Info("The PullThresholdForTopic is changed", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: pc.option.PullThresholdForTopic,
- rlog.LogKeyValueChangedTo: newVal,
- })
- pc.option.PullThresholdForTopic = newVal
- }
-
- if pc.option.PullThresholdSizeForTopic != -1 {
- newVal := pc.option.PullThresholdSizeForTopic / count
- if newVal == 0 {
- newVal = 1
- }
- rlog.Info("The PullThresholdSizeForTopic is changed", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: pc.option.PullThresholdSizeForTopic,
- rlog.LogKeyValueChangedTo: newVal,
- })
- }
- }
- pc.client.SendHeartbeatToAllBrokerWithLock()
-}
-
-func (pc *pushConsumer) validate() {
- internal.ValidateGroup(pc.consumerGroup)
-
- if pc.consumerGroup == internal.DefaultConsumerGroup {
- // TODO FQA
- rlog.Error(fmt.Sprintf("consumerGroup can't equal [%s], please specify another one.", internal.DefaultConsumerGroup), nil)
- }
-
- if len(pc.subscribedTopic) == 0 {
- rlog.Error("number of subscribed topics is 0.", nil)
- }
-
- if pc.option.ConsumeConcurrentlyMaxSpan < 1 || pc.option.ConsumeConcurrentlyMaxSpan > 65535 {
- if pc.option.ConsumeConcurrentlyMaxSpan == 0 {
- pc.option.ConsumeConcurrentlyMaxSpan = 1000
- } else {
- rlog.Error("option.ConsumeConcurrentlyMaxSpan out of range [1, 65535]", nil)
- }
- }
-
- if pc.option.PullThresholdForQueue < 1 || pc.option.PullThresholdForQueue > 65535 {
- if pc.option.PullThresholdForQueue == 0 {
- pc.option.PullThresholdForQueue = 1024
- } else {
- rlog.Error("option.PullThresholdForQueue out of range [1, 65535]", nil)
- }
- }
-
- if pc.option.PullThresholdForTopic < 1 || pc.option.PullThresholdForTopic > 6553500 {
- if pc.option.PullThresholdForTopic == 0 {
- pc.option.PullThresholdForTopic = 102400
- } else {
- rlog.Error("option.PullThresholdForTopic out of range [1, 6553500]", nil)
- }
- }
-
- if pc.option.PullThresholdSizeForQueue < 1 || pc.option.PullThresholdSizeForQueue > 1024 {
- if pc.option.PullThresholdSizeForQueue == 0 {
- pc.option.PullThresholdSizeForQueue = 512
- } else {
- rlog.Error("option.PullThresholdSizeForQueue out of range [1, 1024]", nil)
- }
- }
-
- if pc.option.PullThresholdSizeForTopic < 1 || pc.option.PullThresholdSizeForTopic > 102400 {
- if pc.option.PullThresholdSizeForTopic == 0 {
- pc.option.PullThresholdSizeForTopic = 51200
- } else {
- rlog.Error("option.PullThresholdSizeForTopic out of range [1, 102400]", nil)
- }
- }
-
- if pc.option.PullInterval < 0 || pc.option.PullInterval > 65535*time.Millisecond {
- rlog.Error("option.PullInterval out of range [0, 65535]", nil)
- }
-
- if pc.option.ConsumeMessageBatchMaxSize < 1 || pc.option.ConsumeMessageBatchMaxSize > 1024 {
- if pc.option.ConsumeMessageBatchMaxSize == 0 {
- pc.option.ConsumeMessageBatchMaxSize = 1
- } else {
- rlog.Error("option.ConsumeMessageBatchMaxSize out of range [1, 1024]", nil)
- }
- }
-
- if pc.option.PullBatchSize < 1 || pc.option.PullBatchSize > 1024 {
- if pc.option.PullBatchSize == 0 {
- pc.option.PullBatchSize = 32
- } else {
- rlog.Error("option.PullBatchSize out of range [1, 1024]", nil)
- }
- }
-}
-
-func (pc *pushConsumer) pullMessage(request *PullRequest) {
- rlog.Debug("start a new Pull Message task for PullRequest", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- var sleepTime time.Duration
- pq := request.pq
- go primitive.WithRecover(func() {
- for {
- select {
- case <-pc.done:
- rlog.Info("push consumer close pullMessage.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- default:
- pc.submitToConsume(request.pq, request.mq)
- if request.pq.IsDroppd() {
- rlog.Info("push consumer quit pullMessage for dropped queue.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- }
- })
-
- for {
- NEXT:
- select {
- case <-pc.done:
- rlog.Info("push consumer close message handle.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- default:
- }
-
- if pq.IsDroppd() {
- rlog.Debug("the request was dropped, so stop task", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- return
- }
- if sleepTime > 0 {
- rlog.Debug(fmt.Sprintf("pull MessageQueue: %d sleep %d ms for mq: %v", request.mq.QueueId, sleepTime/time.Millisecond, request.mq), nil)
- time.Sleep(sleepTime)
- }
- // reset time
- sleepTime = pc.option.PullInterval
- pq.lastPullTime.Store(time.Now())
- err := pc.makeSureStateOK()
- if err != nil {
- rlog.Warning("consumer state error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if pc.pause {
- rlog.Debug(fmt.Sprintf("consumer [%s] of [%s] was paused, execute pull request [%s] later",
- pc.option.InstanceName, pc.consumerGroup, request.String()), nil)
- sleepTime = _PullDelayTimeWhenSuspend
- goto NEXT
- }
-
- cachedMessageSizeInMiB := int(pq.cachedMsgSize / Mb)
- if pq.cachedMsgCount > pc.option.PullThresholdForQueue {
- if pc.queueFlowControlTimes%1000 == 0 {
- rlog.Warning("the cached message count exceeds the threshold, so do flow control", map[string]interface{}{
- "PullThresholdForQueue": pc.option.PullThresholdForQueue,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "count": pq.cachedMsgCount,
- "size(MiB)": cachedMessageSizeInMiB,
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- pc.queueFlowControlTimes++
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
-
- if cachedMessageSizeInMiB > pc.option.PullThresholdSizeForQueue {
- if pc.queueFlowControlTimes%1000 == 0 {
- rlog.Warning("the cached message size exceeds the threshold, so do flow control", map[string]interface{}{
- "PullThresholdSizeForQueue": pc.option.PullThresholdSizeForQueue,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "count": pq.cachedMsgCount,
- "size(MiB)": cachedMessageSizeInMiB,
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- pc.queueFlowControlTimes++
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
-
- if !pc.consumeOrderly {
- if pq.getMaxSpan() > pc.option.ConsumeConcurrentlyMaxSpan {
- if pc.queueMaxSpanFlowControlTimes%1000 == 0 {
- rlog.Warning("the queue's messages span too long, so do flow control", map[string]interface{}{
- "ConsumeConcurrentlyMaxSpan": pc.option.ConsumeConcurrentlyMaxSpan,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "maxSpan": pq.getMaxSpan(),
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
- } else {
- if pq.IsLock() {
- if !request.lockedFirst {
- offset := pc.computePullFromWhere(request.mq)
- brokerBusy := offset < request.nextOffset
- rlog.Info("the first time to pull message, so fix offset from broker, offset maybe changed", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- rlog.LogKeyValueChangedFrom: request.nextOffset,
- rlog.LogKeyValueChangedTo: offset,
- "brokerBusy": brokerBusy,
- })
- if brokerBusy {
- rlog.Info("[NOTIFY_ME] the first time to pull message, but pull request offset larger than "+
- "broker consume offset", map[string]interface{}{"offset": offset})
- }
- request.lockedFirst = true
- request.nextOffset = offset
- }
- } else {
- rlog.Info("pull message later because not locked in broker", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
- }
-
- v, exist := pc.subscriptionDataTable.Load(request.mq.Topic)
- if !exist {
- rlog.Info("find the consumer's subscription failed", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
- beginTime := time.Now()
- var (
- commitOffsetEnable bool
- commitOffsetValue int64
- subExpression string
- )
-
- if pc.model == Clustering {
- commitOffsetValue = pc.storage.read(request.mq, _ReadFromMemory)
- if commitOffsetValue > 0 {
- commitOffsetEnable = true
- }
- }
-
- sd := v.(*internal.SubscriptionData)
- classFilter := sd.ClassFilterMode
- if pc.option.PostSubscriptionWhenPull && classFilter {
- subExpression = sd.SubString
- }
-
- sysFlag := buildSysFlag(commitOffsetEnable, true, subExpression != "", classFilter)
-
- pullRequest := &internal.PullMessageRequestHeader{
- ConsumerGroup: pc.consumerGroup,
- Topic: request.mq.Topic,
- QueueId: int32(request.mq.QueueId),
- QueueOffset: request.nextOffset,
- MaxMsgNums: pc.option.PullBatchSize,
- SysFlag: sysFlag,
- CommitOffset: commitOffsetValue,
- SubExpression: _SubAll,
- ExpressionType: string(TAG),
- SuspendTimeoutMillis: 20 * time.Second,
- }
- //
- //if data.ExpType == string(TAG) {
- // pullRequest.SubVersion = 0
- //} else {
- // pullRequest.SubVersion = data.SubVersion
- //}
-
- brokerResult := pc.defaultConsumer.tryFindBroker(request.mq)
- if brokerResult == nil {
- rlog.Warning("no broker found for mq", map[string]interface{}{
- rlog.LogKeyPullRequest: request.mq.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if brokerResult.Slave {
- pullRequest.SysFlag = clearCommitOffsetFlag(pullRequest.SysFlag)
- }
-
- result, err := pc.client.PullMessage(context.Background(), brokerResult.BrokerAddr, pullRequest)
- if err != nil {
- rlog.Warning("pull message from broker error", map[string]interface{}{
- rlog.LogKeyBroker: brokerResult.BrokerAddr,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if result.Status == primitive.PullBrokerTimeout {
- rlog.Warning("pull broker timeout", map[string]interface{}{
- rlog.LogKeyBroker: brokerResult.BrokerAddr,
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- switch result.Status {
- case primitive.PullFound:
- rlog.Debug(fmt.Sprintf("Topic: %s, QueueId: %d found messages.", request.mq.Topic, request.mq.QueueId), nil)
- prevRequestOffset := request.nextOffset
- request.nextOffset = result.NextBeginOffset
-
- rt := time.Now().Sub(beginTime) / time.Millisecond
- pc.stat.increasePullRT(pc.consumerGroup, request.mq.Topic, int64(rt))
-
- pc.processPullResult(request.mq, result, sd)
-
- msgFounded := result.GetMessageExts()
- firstMsgOffset := int64(math.MaxInt64)
- if msgFounded != nil && len(msgFounded) != 0 {
- firstMsgOffset = msgFounded[0].QueueOffset
- pc.stat.increasePullTPS(pc.consumerGroup, request.mq.Topic, len(msgFounded))
- pq.putMessage(msgFounded...)
- }
- if result.NextBeginOffset < prevRequestOffset || firstMsgOffset < prevRequestOffset {
- rlog.Warning("[BUG] pull message result maybe data wrong", map[string]interface{}{
- "nextBeginOffset": result.NextBeginOffset,
- "firstMsgOffset": firstMsgOffset,
- "prevRequestOffset": prevRequestOffset,
- })
- }
- case primitive.PullNoNewMsg, primitive.PullNoMsgMatched:
- request.nextOffset = result.NextBeginOffset
- pc.correctTagsOffset(request)
- case primitive.PullOffsetIllegal:
- rlog.Warning("the pull request offset illegal", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- "result": result.String(),
- })
- request.nextOffset = result.NextBeginOffset
- pq.WithDropped(true)
- time.Sleep(10 * time.Second)
- pc.storage.update(request.mq, request.nextOffset, false)
- pc.storage.persist([]*primitive.MessageQueue{request.mq})
- pc.processQueueTable.Delete(*request.mq)
- rlog.Warning(fmt.Sprintf("fix the pull request offset: %s", request.String()), nil)
- default:
- rlog.Warning(fmt.Sprintf("unknown pull status: %v", result.Status), nil)
- sleepTime = _PullDelayTimeWhenError
- }
- }
-}
-
-func (pc *pushConsumer) correctTagsOffset(pr *PullRequest) {
- if pr.pq.cachedMsgCount <= 0 {
- pc.storage.update(pr.mq, pr.nextOffset, true)
- }
-}
-
-func (pc *pushConsumer) sendMessageBack(brokerName string, msg *primitive.MessageExt, delayLevel int) bool {
- var brokerAddr string
- if len(brokerName) != 0 {
- brokerAddr = pc.defaultConsumer.client.GetNameSrv().FindBrokerAddrByName(brokerName)
- } else {
- brokerAddr = msg.StoreHost
- }
- _, err := pc.client.InvokeSync(context.Background(), brokerAddr, pc.buildSendBackRequest(msg, delayLevel), 3*time.Second)
- if err != nil {
- return false
- }
- return true
-}
-
-func (pc *pushConsumer) buildSendBackRequest(msg *primitive.MessageExt, delayLevel int) *remote.RemotingCommand {
- req := &internal.ConsumerSendMsgBackRequestHeader{
- Group: pc.consumerGroup,
- OriginTopic: msg.Topic,
- Offset: msg.CommitLogOffset,
- DelayLevel: delayLevel,
- OriginMsgId: msg.MsgId,
- MaxReconsumeTimes: pc.getMaxReconsumeTimes(),
- }
-
- return remote.NewRemotingCommand(internal.ReqConsumerSendMsgBack, req, msg.Body)
-}
-
-func (pc *pushConsumer) suspend() {
- pc.pause = true
- rlog.Info(fmt.Sprintf("suspend consumer: %s", pc.consumerGroup), nil)
-}
-
-func (pc *pushConsumer) resume() {
- pc.pause = false
- pc.doBalance()
- rlog.Info(fmt.Sprintf("resume consumer: %s", pc.consumerGroup), nil)
-}
-
-func (pc *pushConsumer) ResetOffset(topic string, table map[primitive.MessageQueue]int64) {
- //topic := cmd.ExtFields["topic"]
- //group := cmd.ExtFields["group"]
- //if topic == "" || group == "" {
- // rlog.Warning("received reset offset command from: %s, but missing params.", from)
- // return
- //}
- //t, err := strconv.ParseInt(cmd.ExtFields["timestamp"], 10, 64)
- //if err != nil {
- // rlog.Warning("received reset offset command from: %s, but parse time error: %s", err.Error())
- // return
- //}
- //rlog.Infof("invoke reset offset operation from broker. brokerAddr=%s, topic=%s, group=%s, timestamp=%v",
- // from, topic, group, t)
- //
- //offsetTable := make(map[MessageQueue]int64, 0)
- //err = json.Unmarshal(cmd.Body, &offsetTable)
- //if err != nil {
- // rlog.Warning("received reset offset command from: %s, but parse offset table: %s", err.Error())
- // return
- //}
- //v, exist := c.consumerMap.Load(group)
- //if !exist {
- // rlog.Infof("[reset-offset] consumer dose not exist. group=%s", group)
- // return
- //}
- pc.suspend()
- defer pc.resume()
-
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- if _, ok := table[mq]; ok && mq.Topic == topic {
- pq.WithDropped(true)
- pq.clear()
- }
- return true
- })
- time.Sleep(10 * time.Second)
- v, exist := pc.topicSubscribeInfoTable.Load(topic)
- if !exist {
- return
- }
- queuesOfTopic := v.([]*primitive.MessageQueue)
- for _, k := range queuesOfTopic {
- if _, ok := table[*k]; ok {
- pc.storage.update(k, table[*k], false)
- v, exist := pc.processQueueTable.Load(k)
- if !exist {
- continue
- }
- pq := v.(*processQueue)
- pc.removeUnnecessaryMessageQueue(k, pq)
- pc.processQueueTable.Delete(k)
- }
- }
-}
-
-func (pc *pushConsumer) removeUnnecessaryMessageQueue(mq *primitive.MessageQueue, pq *processQueue) bool {
- pc.defaultConsumer.removeUnnecessaryMessageQueue(mq, pq)
- if !pc.consumeOrderly || Clustering != pc.model {
- return true
- }
- // TODO orderly
- return true
-}
-
-func (pc *pushConsumer) consumeInner(ctx context.Context, subMsgs []*primitive.MessageExt) (ConsumeResult, error) {
- if len(subMsgs) == 0 {
- return ConsumeRetryLater, errors.New("msg list empty")
- }
-
- f, exist := pc.consumeFunc.Contains(subMsgs[0].Topic)
-
- // fix lost retry message
- if !exist && strings.HasPrefix(subMsgs[0].Topic, internal.RetryGroupTopicPrefix) {
- f, exist = pc.consumeFunc.Contains(subMsgs[0].GetProperty(primitive.PropertyRetryTopic))
- }
-
- if !exist {
- return ConsumeRetryLater, fmt.Errorf("the consume callback missing for topic: %s", subMsgs[0].Topic)
- }
-
- callback, ok := f.(*PushConsumerCallback)
- if !ok {
- return ConsumeRetryLater, fmt.Errorf("the consume callback assert failed for topic: %s", subMsgs[0].Topic)
- }
- if pc.interceptor == nil {
- return callback.f(ctx, subMsgs...)
- } else {
- var container ConsumeResultHolder
- err := pc.interceptor(ctx, subMsgs, &container, func(ctx context.Context, req, reply interface{}) error {
- msgs := req.([]*primitive.MessageExt)
- r, e := callback.f(ctx, msgs...)
-
- realReply := reply.(*ConsumeResultHolder)
- realReply.ConsumeResult = r
-
- msgCtx, _ := primitive.GetConsumerCtx(ctx)
- msgCtx.Success = realReply.ConsumeResult == ConsumeSuccess
- if realReply.ConsumeResult == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- } else {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- }
- return e
- })
- return container.ConsumeResult, err
- }
-}
-
-// resetRetryAndNamespace modify retry message.
-func (pc *pushConsumer) resetRetryAndNamespace(subMsgs []*primitive.MessageExt) {
- groupTopic := internal.RetryGroupTopicPrefix + pc.consumerGroup
- beginTime := time.Now()
- for idx := range subMsgs {
- msg := subMsgs[idx]
- retryTopic := msg.GetProperty(primitive.PropertyRetryTopic)
- if retryTopic == "" && groupTopic == msg.Topic {
- msg.Topic = retryTopic
- }
- subMsgs[idx].WithProperty(primitive.PropertyConsumeStartTime, strconv.FormatInt(
- beginTime.UnixNano()/int64(time.Millisecond), 10))
- }
-}
-
-func (pc *pushConsumer) consumeMessageCurrently(pq *processQueue, mq *primitive.MessageQueue) {
- msgs := pq.getMessages()
- if msgs == nil {
- return
- }
- for count := 0; count < len(msgs); count++ {
- var subMsgs []*primitive.MessageExt
- if count+pc.option.ConsumeMessageBatchMaxSize > len(msgs) {
- subMsgs = msgs[count:]
- count = len(msgs)
- } else {
- next := count + pc.option.ConsumeMessageBatchMaxSize
- subMsgs = msgs[count:next]
- count = next - 1
- }
- go primitive.WithRecover(func() {
- RETRY:
- if pq.IsDroppd() {
- rlog.Info("the message queue not be able to consume, because it was dropped", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
-
- beginTime := time.Now()
- pc.resetRetryAndNamespace(subMsgs)
- var result ConsumeResult
-
- var err error
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: subMsgs,
- }
- ctx := context.Background()
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
- concurrentCtx := primitive.NewConsumeConcurrentlyContext()
- concurrentCtx.MQ = *mq
- ctx = primitive.WithConcurrentlyCtx(ctx, concurrentCtx)
-
- result, err = pc.consumeInner(ctx, subMsgs)
-
- consumeRT := time.Now().Sub(beginTime)
- if err != nil {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.ExceptionReturn)
- } else if consumeRT >= pc.option.ConsumeTimeout {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.TimeoutReturn)
- } else if result == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- } else if result == ConsumeRetryLater {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- }
-
- pc.stat.increaseConsumeRT(pc.consumerGroup, mq.Topic, int64(consumeRT/time.Millisecond))
-
- if !pq.IsDroppd() {
- msgBackFailed := make([]*primitive.MessageExt, 0)
- if result == ConsumeSuccess {
- pc.stat.increaseConsumeOKTPS(pc.consumerGroup, mq.Topic, len(subMsgs))
- } else {
- pc.stat.increaseConsumeFailedTPS(pc.consumerGroup, mq.Topic, len(subMsgs))
- if pc.model == BroadCasting {
- for i := 0; i < len(subMsgs); i++ {
- rlog.Warning("BROADCASTING, the message consume failed, drop it", map[string]interface{}{
- "message": subMsgs[i],
- })
- }
- } else {
- for i := 0; i < len(subMsgs); i++ {
- msg := subMsgs[i]
- if !pc.sendMessageBack(mq.BrokerName, msg, concurrentCtx.DelayLevelWhenNextConsume) {
- msg.ReconsumeTimes += 1
- msgBackFailed = append(msgBackFailed, msg)
- }
- }
- }
- }
-
- offset := pq.removeMessage(subMsgs...)
-
- if offset >= 0 && !pq.IsDroppd() {
- pc.storage.update(mq, int64(offset), true)
- }
- if len(msgBackFailed) > 0 {
- subMsgs = msgBackFailed
- time.Sleep(5 * time.Second)
- goto RETRY
- }
- } else {
- rlog.Warning("processQueue is dropped without process consume result.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- "message": subMsgs,
- })
- }
- })
- }
-}
-
-func (pc *pushConsumer) consumeMessageOrderly(pq *processQueue, mq *primitive.MessageQueue) {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- return
- }
-
- lock := pc.queueLock.fetchLock(*mq)
- lock.Lock()
- defer lock.Unlock()
- if pc.model == BroadCasting || (pq.IsLock() && !pq.isLockExpired()) {
- beginTime := time.Now()
-
- continueConsume := true
- for continueConsume {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- break
- }
- if pc.model == Clustering {
- if !pq.IsLock() {
- rlog.Warning("the message queue not locked, so consume later", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pc.tryLockLaterAndReconsume(mq, 10)
- return
- }
- if pq.isLockExpired() {
- rlog.Warning("the message queue lock expired, so consume later", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pc.tryLockLaterAndReconsume(mq, 10)
- return
- }
- }
- interval := time.Now().Sub(beginTime)
- if interval > pc.option.MaxTimeConsumeContinuously {
- time.Sleep(10 * time.Millisecond)
- return
- }
- batchSize := pc.option.ConsumeMessageBatchMaxSize
- msgs := pq.takeMessages(batchSize)
-
- pc.resetRetryAndNamespace(msgs)
-
- if len(msgs) == 0 {
- continueConsume = false
- break
- }
-
- // TODO: add message consumer hook
- beginTime = time.Now()
-
- ctx := context.Background()
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: msgs,
- }
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
-
- orderlyCtx := primitive.NewConsumeOrderlyContext()
- orderlyCtx.MQ = *mq
- ctx = primitive.WithOrderlyCtx(ctx, orderlyCtx)
-
- pq.lockConsume.Lock()
- result, _ := pc.consumeInner(ctx, msgs)
- pq.lockConsume.Unlock()
-
- if result == Rollback || result == SuspendCurrentQueueAMoment {
- rlog.Warning("consumeMessage Orderly return not OK", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- "messages": msgs,
- rlog.LogKeyMessageQueue: mq,
- })
- }
-
- // jsut put consumeResult in consumerMessageCtx
- //interval = time.Now().Sub(beginTime)
- //consumeReult := SuccessReturn
- //if interval > pc.option.ConsumeTimeout {
- // consumeReult = TimeoutReturn
- //} else if SuspendCurrentQueueAMoment == result {
- // consumeReult = FailedReturn
- //} else if ConsumeSuccess == result {
- // consumeReult = SuccessReturn
- //}
-
- // process result
- commitOffset := int64(-1)
- if pc.option.AutoCommit {
- switch result {
- case Commit, Rollback:
- rlog.Warning("the message queue consume result is illegal, we think you want to ack these message: %v", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- })
- case ConsumeSuccess:
- commitOffset = pq.commit()
- case SuspendCurrentQueueAMoment:
- if pc.checkReconsumeTimes(msgs) {
- pq.makeMessageToCosumeAgain(msgs...)
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- } else {
- commitOffset = pq.commit()
- }
- default:
- }
- } else {
- switch result {
- case ConsumeSuccess:
- case Commit:
- commitOffset = pq.commit()
- case Rollback:
- // pq.rollback
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- case SuspendCurrentQueueAMoment:
- if pc.checkReconsumeTimes(msgs) {
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- }
- default:
- }
- }
- if commitOffset > 0 && !pq.IsDroppd() {
- _ = pc.updateOffset(mq, commitOffset)
- }
- }
- } else {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- pc.tryLockLaterAndReconsume(mq, 100)
- }
-}
-
-func (pc *pushConsumer) checkReconsumeTimes(msgs []*primitive.MessageExt) bool {
- suspend := false
- if len(msgs) != 0 {
- maxReconsumeTimes := pc.getOrderlyMaxReconsumeTimes()
- for _, msg := range msgs {
- if msg.ReconsumeTimes > maxReconsumeTimes {
- rlog.Warning(fmt.Sprintf("msg will be send to retry topic due to ReconsumeTimes > %d, \n", maxReconsumeTimes), nil)
- msg.WithProperty("RECONSUME_TIME", strconv.Itoa(int(msg.ReconsumeTimes)))
- if !pc.sendMessageBack("", msg, -1) {
- suspend = true
- msg.ReconsumeTimes += 1
- }
- } else {
- suspend = true
- msg.ReconsumeTimes += 1
- }
- }
- }
- return suspend
-}
-
-func (pc *pushConsumer) getOrderlyMaxReconsumeTimes() int32 {
- if pc.option.MaxReconsumeTimes == -1 {
- return math.MaxInt32
- } else {
- return pc.option.MaxReconsumeTimes
- }
-}
-
-func (pc *pushConsumer) getMaxReconsumeTimes() int32 {
- if pc.option.MaxReconsumeTimes == -1 {
- return 16
- } else {
- return pc.option.MaxReconsumeTimes
- }
-}
-
-func (pc *pushConsumer) tryLockLaterAndReconsume(mq *primitive.MessageQueue, delay int64) {
- time.Sleep(time.Duration(delay) * time.Millisecond)
- if pc.lock(mq) == true {
- pc.submitConsumeRequestLater(10)
- } else {
- pc.submitConsumeRequestLater(3000)
- }
-}
-
-func (pc *pushConsumer) submitConsumeRequestLater(suspendTimeMillis int64) {
- if suspendTimeMillis == -1 {
- suspendTimeMillis = int64(pc.option.SuspendCurrentQueueTimeMillis / time.Millisecond)
- }
- if suspendTimeMillis < 10 {
- suspendTimeMillis = 10
- } else if suspendTimeMillis > 30000 {
- suspendTimeMillis = 30000
- }
- time.Sleep(time.Duration(suspendTimeMillis) * time.Millisecond)
-}
diff --git a/consumer/push_consumer_test.go b/consumer/push_consumer_test.go
deleted file mode 100644
index 78bc1f7..0000000
--- a/consumer/push_consumer_test.go
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func mockB4Start(c *pushConsumer) {
- c.topicSubscribeInfoTable.Store("TopicTest", []*primitive.MessageQueue{})
-}
-
-func TestStart(t *testing.T) {
- Convey("test Start method", t, func() {
- c, _ := NewPushConsumer(
- WithGroupName("testGroup"),
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithConsumerModel(BroadCasting),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- client := internal.NewMockRMQClient(ctrl)
- c.client = client
-
- err := c.Subscribe("TopicTest", MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return ConsumeSuccess, nil
- })
-
- _, exists := c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeTrue)
-
- err = c.Unsubscribe("TopicTest")
- So(err, ShouldBeNil)
- _, exists = c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeFalse)
-
- err = c.Subscribe("TopicTest", MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return ConsumeSuccess, nil
- })
-
- _, exists = c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeTrue)
-
- client.EXPECT().ClientID().Return("127.0.0.1@DEFAULT")
- client.EXPECT().Start().Return()
- client.EXPECT().RegisterConsumer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().UpdateTopicRouteInfo().AnyTimes().Return()
-
- Convey("test topic route info not found", func() {
- client.EXPECT().Shutdown().Return()
- err = c.Start()
- So(err.Error(), ShouldContainSubstring, "route info not found")
- })
-
- Convey("test topic route info found", func() {
- client.EXPECT().RebalanceImmediately().Return()
- client.EXPECT().CheckClientInBroker().Return()
- client.EXPECT().SendHeartbeatToAllBrokerWithLock().Return()
- mockB4Start(c)
- err = c.Start()
- So(err, ShouldBeNil)
- })
- })
-}
diff --git a/consumer/statistics.go b/consumer/statistics.go
deleted file mode 100644
index e9d5d79..0000000
--- a/consumer/statistics.go
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "container/list"
- "fmt"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type StatsManager struct {
- startOnce sync.Once
- closeOnce sync.Once
- topicAndGroupConsumeOKTPS *statsItemSet
- topicAndGroupConsumeRT *statsItemSet
- topicAndGroupConsumeFailedTPS *statsItemSet
- topicAndGroupPullTPS *statsItemSet
- topicAndGroupPullRT *statsItemSet
-}
-
-func NewStatsManager() *StatsManager {
- mgr := &StatsManager{}
- mgr.topicAndGroupConsumeOKTPS = newStatsItemSet("CONSUME_OK_TPS")
- mgr.topicAndGroupConsumeRT = newStatsItemSet("CONSUME_RT")
- mgr.topicAndGroupConsumeFailedTPS = newStatsItemSet("CONSUME_FAILED_TPS")
- mgr.topicAndGroupPullTPS = newStatsItemSet("PULL_TPS")
- mgr.topicAndGroupPullRT = newStatsItemSet("PULL_RT")
- return mgr
-}
-
-type ConsumeStatus struct {
- PullRT float64
- PullTPS float64
- ConsumeRT float64
- ConsumeOKTPS float64
- ConsumeFailedTPS float64
- ConsumeFailedMsgs int64
-}
-
-func (mgr *StatsManager) increasePullRT(group, topic string, rt int64) {
- mgr.topicAndGroupPullRT.addValue(topic+"@"+group, rt, 1)
-}
-
-func (mgr *StatsManager) increasePullTPS(group, topic string, msgs int) {
- mgr.topicAndGroupPullTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) increaseConsumeRT(group, topic string, rt int64) {
- mgr.topicAndGroupConsumeRT.addValue(topic+"@"+group, rt, 1)
-}
-
-func (mgr *StatsManager) increaseConsumeOKTPS(group, topic string, msgs int) {
- mgr.topicAndGroupConsumeOKTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) increaseConsumeFailedTPS(group, topic string, msgs int) {
- mgr.topicAndGroupConsumeFailedTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) GetConsumeStatus(group, topic string) ConsumeStatus {
- cs := ConsumeStatus{}
- ss := mgr.getPullRT(group, topic)
- cs.PullTPS = ss.tps
-
- ss = mgr.getPullTPS(group, topic)
- cs.PullTPS = ss.tps
-
- ss = mgr.getConsumeRT(group, topic)
- cs.ConsumeRT = ss.avgpt
-
- ss = mgr.getConsumeOKTPS(group, topic)
- cs.ConsumeOKTPS = ss.tps
-
- ss = mgr.getConsumeFailedTPS(group, topic)
-
- cs.ConsumeFailedTPS = ss.tps
-
- ss = mgr.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + group)
- cs.ConsumeFailedMsgs = ss.sum
- return cs
-}
-
-func (mgr *StatsManager) ShutDownStat() {
- mgr.closeOnce.Do(func() {
- close(mgr.topicAndGroupConsumeOKTPS.closed)
- close(mgr.topicAndGroupConsumeRT.closed)
- close(mgr.topicAndGroupConsumeFailedTPS.closed)
- close(mgr.topicAndGroupPullTPS.closed)
- close(mgr.topicAndGroupPullRT.closed)
- })
-}
-
-func (mgr *StatsManager) getPullRT(group, topic string) statsSnapshot {
- return mgr.topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getPullTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupPullTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getConsumeRT(group, topic string) statsSnapshot {
- ss := mgr.topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group)
- if ss.sum == 0 {
- return mgr.topicAndGroupConsumeRT.getStatsDataInHour(topic + "@" + group)
- }
- return ss
-}
-
-func (mgr *StatsManager) getConsumeOKTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupConsumeOKTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getConsumeFailedTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupConsumeFailedTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-var csListLock sync.Mutex
-
-func computeStatsData(csList *list.List) statsSnapshot {
- csListLock.Lock()
- defer csListLock.Unlock()
- tps, avgpt, sum := 0.0, 0.0, int64(0)
- if csList.Len() > 0 {
- first := csList.Front().Value.(callSnapshot)
- last := csList.Back().Value.(callSnapshot)
- sum = last.value - first.value
- tps = float64(sum*1000.0) / float64(last.timestamp-first.timestamp)
- timesDiff := last.time - first.time
- if timesDiff > 0 {
- avgpt = float64(sum*1.0) / float64(timesDiff)
- }
- }
- return statsSnapshot{
- tps: tps,
- avgpt: avgpt,
- sum: sum,
- }
-}
-
-type statsItemSet struct {
- statsName string
- statsItemTable sync.Map
- closed chan struct{}
-}
-
-func newStatsItemSet(statsName string) *statsItemSet {
- sis := &statsItemSet{
- statsName: statsName,
- closed: make(chan struct{}),
- }
- sis.init()
- return sis
-}
-
-func (sis *statsItemSet) init() {
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(10 * time.Second)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInSeconds()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(10 * time.Minute)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInMinutes()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInHour()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextMinutesTime().Sub(time.Now()))
- ticker := time.NewTicker(time.Minute)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtMinutes()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextHourTime().Sub(time.Now()))
- ticker := time.NewTicker(time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtHour()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextMonthTime().Sub(time.Now()))
- ticker := time.NewTicker(24 * time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtDay()
- }
- }
- })
-}
-
-func (sis *statsItemSet) samplingInSeconds() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInSeconds()
- return true
- })
-}
-
-func (sis *statsItemSet) samplingInMinutes() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInMinutes()
- return true
- })
-}
-
-func (sis *statsItemSet) samplingInHour() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInHour()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtMinutes() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtMinutes()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtHour() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtHour()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtDay() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtDay()
- return true
- })
-}
-
-func (sis *statsItemSet) addValue(key string, incValue, incTimes int64) {
- si := sis.getAndCreateStateItem(key)
- atomic.AddInt64(&si.value, incValue)
- atomic.AddInt64(&si.times, incTimes)
-}
-
-func (sis *statsItemSet) getAndCreateStateItem(key string) *statsItem {
- if val, ok := sis.statsItemTable.Load(key); ok {
- return val.(*statsItem)
- } else {
- si := newStatsItem(sis.statsName, key)
- sis.statsItemTable.Store(key, si)
- return si
- }
-}
-
-func (sis *statsItemSet) getStatsDataInMinute(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInMinute()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsDataInHour(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInHour()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsDataInDay(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInDay()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsItem(key string) *statsItem {
- val, _ := sis.statsItemTable.Load(key)
- return val.(*statsItem)
-}
-
-type statsItem struct {
- value int64
- times int64
- csListMinute *list.List
- csListHour *list.List
- csListDay *list.List
- statsName string
- statsKey string
- csListMinuteLock sync.Mutex
- csListHourLock sync.Mutex
- csListDayLock sync.Mutex
-}
-
-func (si *statsItem) getStatsDataInMinute() statsSnapshot {
- return computeStatsData(si.csListMinute)
-}
-
-func (si *statsItem) getStatsDataInHour() statsSnapshot {
- return computeStatsData(si.csListHour)
-}
-
-func (si *statsItem) getStatsDataInDay() statsSnapshot {
- return computeStatsData(si.csListDay)
-}
-
-func newStatsItem(statsName, statsKey string) *statsItem {
- return &statsItem{
- statsName: statsName,
- statsKey: statsKey,
- csListMinute: list.New(),
- csListHour: list.New(),
- csListDay: list.New(),
- }
-}
-
-func (si *statsItem) samplingInSeconds() {
- si.csListMinuteLock.Lock()
- defer si.csListMinuteLock.Unlock()
- si.csListMinute.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListMinute.Len() > 7 {
- si.csListMinute.Remove(si.csListMinute.Front())
- }
-}
-
-func (si *statsItem) samplingInMinutes() {
- si.csListHourLock.Lock()
- defer si.csListHourLock.Unlock()
- si.csListHour.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListHour.Len() > 7 {
- si.csListHour.Remove(si.csListHour.Front())
- }
-}
-
-func (si *statsItem) samplingInHour() {
- si.csListDayLock.Lock()
- defer si.csListDayLock.Unlock()
- si.csListDay.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListDay.Len() > 25 {
- si.csListHour.Remove(si.csListDay.Front())
- }
-}
-
-func (si *statsItem) printAtMinutes() {
- ss := computeStatsData(si.csListMinute)
- rlog.Info("Stats In One Minute, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func (si *statsItem) printAtHour() {
- ss := computeStatsData(si.csListHour)
- rlog.Info("Stats In One Hour, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func (si *statsItem) printAtDay() {
- ss := computeStatsData(si.csListDay)
- rlog.Info("Stats In One Day, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func nextMinutesTime() time.Time {
- now := time.Now()
- m, _ := time.ParseDuration("1m")
- return now.Add(m)
-}
-
-func nextHourTime() time.Time {
- now := time.Now()
- m, _ := time.ParseDuration("1h")
- return now.Add(m)
-}
-
-func nextMonthTime() time.Time {
- now := time.Now()
- return now.AddDate(0, 1, 0)
-}
-
-type callSnapshot struct {
- timestamp int64
- time int64
- value int64
-}
-
-type statsSnapshot struct {
- sum int64
- tps float64
- avgpt float64
-}
diff --git a/consumer/statistics_test.go b/consumer/statistics_test.go
deleted file mode 100644
index 4836141..0000000
--- a/consumer/statistics_test.go
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "testing"
- "time"
-)
-
-func almostEqual(a, b float64) bool {
- diff := abs(a - b)
- return diff/a < 0.01
-}
-
-func abs(a float64) float64 {
- if a > 0 {
- return a
- }
- return -a
-}
-
-func TestNextMinuteTime(t *testing.T) {
- nextMinute := nextMinutesTime()
- minuteElapse := nextMinute.Sub(time.Now()).Minutes()
- if !almostEqual(minuteElapse, 1.0) {
- t.Errorf("wrong next one minute. want=%f, got=%f", 1.0, minuteElapse)
- }
-}
-
-func TestNextHourTime(t *testing.T) {
- nextHour := nextHourTime()
- hourElapse := nextHour.Sub(time.Now()).Hours()
- if !almostEqual(hourElapse, 1.0) {
- t.Errorf("wrong next one hour. want=%f, got=%f", 1.0, hourElapse)
- }
-}
-
-func TestIncreasePullRTGetPullRT(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
-
- tests := []struct {
- RT int64
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increasePullRT("rocketmq", "default", tt.RT)
- mgr.topicAndGroupPullRT.samplingInSeconds()
- snapshot := mgr.getPullRT("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Pull RT sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-//func TestIncreaseConsumeRTGetConsumeRT(t *testing.T) {
-// ShutDownStat()
-// tests := []struct {
-// RT int64
-// ExpectSum int64
-// }{
-// {1, 0},
-// {1, 1},
-// {1, 2},
-// {1, 3},
-// {1, 4},
-// {1, 5},
-// {1, 6},
-// {1, 6},
-// }
-// for _, tt := range tests {
-// increaseConsumeRT("rocketmq", "default", tt.RT)
-// topicAndGroupConsumeRT.samplingInMinutes()
-// snapshot := getConsumeRT("rocketmq", "default")
-// if snapshot.sum != tt.ExpectSum {
-// t.Errorf("wrong consume RT sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
-// }
-// }
-//}
-
-func TestIncreasePullTPSGetPullTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increasePullTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupPullTPS.samplingInSeconds()
- snapshot := mgr.getPullTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Pull TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestIncreaseConsumeOKTPSGetConsumeOKTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increaseConsumeOKTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupConsumeOKTPS.samplingInSeconds()
- snapshot := mgr.getConsumeOKTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Consume OK TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestIncreaseConsumeFailedTPSGetConsumeFailedTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increaseConsumeFailedTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupConsumeFailedTPS.samplingInSeconds()
- snapshot := mgr.getConsumeFailedTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Consume Failed TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestGetConsumeStatus(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- group, topic := "rocketmq", "default"
-
- tests := []struct {
- RT int
- ExpectFailMessage int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- }
- for _, tt := range tests {
- mgr.increasePullRT(group, topic, int64(tt.RT))
- mgr.increasePullTPS(group, topic, tt.RT)
- mgr.increaseConsumeRT(group, topic, int64(tt.RT))
- mgr.increaseConsumeOKTPS(group, topic, tt.RT)
- mgr.increaseConsumeFailedTPS(group, topic, tt.RT)
- mgr.topicAndGroupPullRT.samplingInSeconds()
- mgr.topicAndGroupPullTPS.samplingInSeconds()
- mgr.topicAndGroupConsumeRT.samplingInMinutes()
- mgr.topicAndGroupConsumeOKTPS.samplingInSeconds()
- mgr.topicAndGroupConsumeFailedTPS.samplingInMinutes()
- status := mgr.GetConsumeStatus(group, topic)
- if status.ConsumeFailedMsgs != tt.ExpectFailMessage {
- t.Errorf("wrong ConsumeFailedMsg. want=0, got=%d", status.ConsumeFailedMsgs)
- }
- }
-}
diff --git a/consumer/strategy.go b/consumer/strategy.go
deleted file mode 100644
index 4a07928..0000000
--- a/consumer/strategy.go
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "strings"
-
- "github.com/stathat/consistent"
-
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-// Strategy Algorithm for message allocating between consumers
-// An allocate strategy proxy for based on machine room nearside priority. An actual allocate strategy can be
-// specified.
-//
-// If any consumer is alive in a machine room, the message queue of the broker which is deployed in the same machine
-// should only be allocated to those. Otherwise, those message queues can be shared along all consumers since there are
-// no alive consumer to monopolize them.
-//
-// Average Hashing queue algorithm
-// Cycle average Hashing queue algorithm
-// Use Message QueueID specified
-// Computer room Hashing queue algorithm, such as Alipay logic room
-// Consistent Hashing queue algorithm
-
-type AllocateStrategy func(string, string, []*primitive.MessageQueue, []string) []*primitive.MessageQueue
-
-func AllocateByAveragely(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- mqSize := len(mqAll)
- cidSize := len(cidAll)
- mod := mqSize % cidSize
-
- var averageSize int
- if mqSize <= cidSize {
- averageSize = 1
- } else {
- if mod > 0 && index < mod {
- averageSize = mqSize/cidSize + 1
- } else {
- averageSize = mqSize / cidSize
- }
- }
-
- var startIndex int
- if mod > 0 && index < mod {
- startIndex = index * averageSize
- } else {
- startIndex = index*averageSize + mod
- }
-
- num := utils.MinInt(averageSize, mqSize-startIndex)
- result := make([]*primitive.MessageQueue, 0)
- for i := 0; i < num; i++ {
- result = append(result, mqAll[(startIndex+i)%mqSize])
- }
- return result
-}
-
-func AllocateByAveragelyCircle(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- result := make([]*primitive.MessageQueue, 0)
- for i := index; i < len(mqAll); i++ {
- if i%len(cidAll) == index {
- result = append(result, mqAll[i])
- }
- }
- return result
-}
-
-// TODO
-func AllocateByMachineNearby(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- return AllocateByAveragely(consumerGroup, currentCID, mqAll, cidAll)
-}
-
-func AllocateByConfig(list []*primitive.MessageQueue) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- return list
- }
-}
-
-func AllocateByMachineRoom(consumeridcs []string) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- var premqAll []*primitive.MessageQueue
- for _, mq := range mqAll {
- temp := strings.Split(mq.BrokerName, "@")
- if len(temp) == 2 {
- for _, idc := range consumeridcs {
- if idc == temp[0] {
- premqAll = append(premqAll, mq)
- }
- }
- }
- }
-
- mod := len(premqAll) / len(cidAll)
- rem := len(premqAll) % len(cidAll)
- startIndex := mod * index
- endIndex := startIndex + mod
-
- result := make([]*primitive.MessageQueue, 0)
- for i := startIndex; i < endIndex; i++ {
- result = append(result, mqAll[i])
- }
- if rem > index {
- result = append(result, premqAll[index+mod*len(cidAll)])
- }
- return result
- }
-}
-
-func AllocateByConsistentHash(virtualNodeCnt int) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- c := consistent.New()
- c.NumberOfReplicas = virtualNodeCnt
- for _, cid := range cidAll {
- c.Add(cid)
- }
-
- result := make([]*primitive.MessageQueue, 0)
- for _, mq := range mqAll {
- clientNode, err := c.Get(mq.String())
- if err != nil {
- rlog.Warning("[BUG] AllocateByConsistentHash err: %s", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- }
- if currentCID == clientNode {
- result = append(result, mq)
- }
- }
- return result
- }
-}
diff --git a/consumer/strategy_test.go b/consumer/strategy_test.go
deleted file mode 100644
index d521b4b..0000000
--- a/consumer/strategy_test.go
+++ /dev/null
@@ -1,487 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func TestAllocateByAveragely(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- Convey("When params is empty", func() {
- result := AllocateByAveragely("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragely("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragely("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 5,
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByAveragely should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := AllocateByAveragely("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByAveragelyCircle(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- Convey("When params is empty", func() {
- result := AllocateByAveragelyCircle("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragelyCircle("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragelyCircle("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 4,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- },
- {
- QueueId: 4,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- },
- {
- QueueId: 5,
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByAveragelyCircle should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := AllocateByAveragelyCircle("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByConfig(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- strategy := AllocateByConfig(queues)
- result := strategy("testGroup", "192.168.24.1@default", queues, []string{"192.168.24.1@default", "192.168.24.2@default"})
- So(result, ShouldResemble, queues)
- })
-}
-
-func TestAllocateByMachineRoom(t *testing.T) {
- Convey("Given some consumer IDCs with a starting value", t, func() {
- idcs := []string{"192.168.24.1", "192.168.24.2"}
- strategy := AllocateByMachineRoom(idcs)
-
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 2,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 5,
- BrokerName: "192.168.24.3@defaultName",
- },
- }
-
- Convey("When params is empty", func() {
- result := strategy("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByMachineRoom should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := strategy("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByConsistentHash(t *testing.T) {
- Convey("Given virtualNodeCnt with a starting value", t, func() {
- virtualNodeCnt := 10
- strategy := AllocateByConsistentHash(virtualNodeCnt)
-
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 2,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 5,
- BrokerName: "192.168.24.3@defaultName",
- },
- }
-
- Convey("When params is empty", func() {
- result := strategy("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.3@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- },
- }
-
- Convey("observe the result of AllocateByMachineRoom", func() {
- for _, value := range cases {
- result := strategy("testGroup", value.currentCid, value.mqAll, value.cidAll)
- rlog.Info("Result Of AllocateByMachineRoom", map[string]interface{}{
- "currentCid": value.currentCid,
- "cidAll": value.cidAll,
- "allocateResult": result,
- })
- }
- })
- })
-}
diff --git a/docs/Introduction.md b/docs/Introduction.md
deleted file mode 100644
index 011603a..0000000
--- a/docs/Introduction.md
+++ /dev/null
@@ -1,130 +0,0 @@
-## How to use
-
-### go mod
-```
-require (
- github.com/apache/rocketmq-client-go/v2 v2.1.0-rc3
-)
-```
-
-### Set Logger
-Go Client define the `Logger` interface for log output, user can specify implementation of private.
-in default, client use `logrus`.
-```
-rlog.SetLogger(Logger)
-```
-
-### Send message
-#### Interface
-```
-Producer interface {
- Start() error
- Shutdown() error
- SendSync(context.Context, *primitive.Message) (*internal.SendResult, error)
- SendOneWay(context.Context, *primitive.Message) error
-}
-```
-
-#### Examples
-- create a new `Producer` instance
-```
-p, err := rocketmq.NewProducer(
- producer.WithNameServer(endPoint),
- //producer.WithNsResolver(primitive.NewPassthroughResolver(endPoint)),
- producer.WithRetry(2),
- producer.WithGroupName("GID_xxxxxx"),
- )
-```
-
-- start the producer
-```go
-err := p.Start()
-```
-
-- send message with sync
-```
-result, err := p.SendSync(context.Background(), &primitive.Message{
- Topic: "test",
- Body: []byte("Hello RocketMQ Go Client!"),
-})
-
-// do something with result
-```
-
-- or send message with oneway
-```
-err := p.SendOneWay(context.Background(), &primitive.Message{
- Topic: "test",
- Body: []byte("Hello RocketMQ Go Client!"),
-})
-```
-Full examples: [producer](../examples/producer)
-
-### Consume Message
-now only support `PushConsumer`
-
-#### Interface
-```
-PushConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be sync to broker before process exit
- Shutdown() error
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (consumer.ConsumeResult, error)) error
-}
-```
-
-#### Usage
-- Create a `PushConsumer` instance
-```
-c, err := rocketmq.NewPushConsumer(
- consumer.WithNameServer(endPoint),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithGroupName("GID_XXXXXX"),
- )
-```
-
-- Subscribe a topic(only support one topic now), and define your consuming function
-```
-err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx *consumer.ConsumeMessageContext,
- msgs []*primitive.MessageExt) (consumer.ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return consumer.ConsumeSuccess, nil
-})
-```
-- start the consumer(**NOTE: MUST after subscribe**)
-```
-err = c.Start()
-```
-
-Full examples: [consumer](../examples/consumer)
-
-
-### Admin: Topic Operation
-
-#### Examples
-- create topic
-```
-testAdmin, err := admin.NewAdmin(admin.WithResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})))
-err = testAdmin.CreateTopic(
- context.Background(),
- admin.WithTopicCreate("newTopic"),
- admin.WithBrokerAddrCreate("127.0.0.1:10911"),
-)
-```
-
-- delete topic
-`ClusterName` not supported yet
-```
-err = testAdmin.DeleteTopic(
- context.Background(),
- admin.WithTopicDelete("newTopic"),
- //admin.WithBrokerAddrDelete("127.0.0.1:10911"), //optional
- //admin.WithNameSrvAddr(nameSrvAddr), //optional
-)
-```
\ No newline at end of file
diff --git a/docs/client-design.gliffy b/docs/client-design.gliffy
deleted file mode 100644
index 418f4d8..0000000
--- a/docs/client-design.gliffy
+++ /dev/null
@@ -1 +0,0 @@
-{"contentType":"application/gliffy+json","version":"1.1","metadata":{"title":"untitled","revision":0,"exportBorder":false},"embeddedResources":{"index":0,"resources":[]},"stage":{"objects":[{"x":31,"y":44,"rotation":0,"id":57,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":27,"lockAspectRatio":false,"lockShape":false,"order":57,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\ [...]
\ No newline at end of file
diff --git a/docs/feature.md b/docs/feature.md
deleted file mode 100644
index eed3e17..0000000
--- a/docs/feature.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# Feature
-
-## Producer
-
-### MessageType
-- [x] NormalMessage
-- [ ] TransactionMessage
-- [ ] DelayMessage
-
-### SendWith
-- [x] Sync
-- [ ] Async
-- [x] OneWay
-
-### Other
-- [ ] Config
-- [ ] MessageId Generate
-- [ ] CompressMsg
-- [ ] LoadBalance
-- [ ] DefaultTopic
-- [ ] VipChannel
-- [ ] Retry
-- [ ] Hook
-- [ ] CheckRequestQueue
-- [ ] MQFaultStrategy
-
-## Consumer
-
-### ReceiveType
-- [x] Push
-- [ ] Pull
-
-### ConsumingType
-- [x] Concurrently
-- [ ] Orderly
-
-### MessageModel
-- [x] CLUSTERING
-- [x] BROADCASTING
-
-### AllocateMessageQueueStrategy
-- [x] AllocateMessageQueueAveragely
-- [x] AllocateMessageQueueAveragelyByCircle
-- [X] AllocateMessageQueueByConfig
-- [X] AllocateMessageQueueByMachineRoom
-
-### Other
-- [x] Rebalance
-- [x] Flow Control
-- [ ] compress
-- [x] ConsumeFromWhere
-- [ ] Retry(sendMessageBack)
-- [ ] Hook
-
-## Common
-- [ ] PollNameServer
-- [x] Heartbeat
-- [x] UpdateTopicRouteInfoFromNameServer
-- [ ] CleanOfflineBroker
-- [ ] ClearExpiredMessage(form consumer consumeMessageService)
-
-## Remoting
-- [x] API
- - [x] InvokeSync
- - [x] InvokeAsync
- - [x] InvokeOneWay
-- [x] Serialize
- - [x] JSON
- - [x] ROCKETMQ
-- [ ] Other
- - [ ] VIPChannel
- - [ ] RPCHook
-
-## Admin
-
-### Topic/Cluster
-- [x] updateTopic
-- [x] deleteTopic
-- [ ] updateSubGroup
-- [ ] deleteSubGroup
-- [ ] updateBrokerConfig
-- [ ] updateTopicPerm
-- [ ] listTopic
-- [ ] topicRoute
-- [ ] topicStatus
-- [ ] topicClusterList
\ No newline at end of file
diff --git a/docs/images/client-design.png b/docs/images/client-design.png
deleted file mode 100644
index 97b86ef..0000000
Binary files a/docs/images/client-design.png and /dev/null differ
diff --git a/docs/zh/native-design_zh.md b/docs/zh/native-design_zh.md
deleted file mode 100644
index dd407a4..0000000
--- a/docs/zh/native-design_zh.md
+++ /dev/null
@@ -1,114 +0,0 @@
-# RocketMQ Go Client Design Draft
-
-## Architecture
-
-### Overview
-
-
-### Description
-在RocketMQ Java Client的实现里面，代码耦合了大量的admin方面的功能， 其为了尽可能的提高代码复用率，代码的依赖关系较为复杂、接口的设计比
-较重、语义的界限不够清晰。因此，为了避免简单的对Java代码进行翻译，故Go客户端进行了重新的设计，剥离了admin相关的逻辑。整体如上面的图所示，在逻辑层次上，按照请求
-顺序，从下到上总共分为三层：
-- remote层：client网络通信和私有协议层，将到每个到服务端(NameServer或Broker)的连接实体抽象为一个client结构，并在这里实现了网络数据的
-序列/反序列化。
-- 公共层：由于remote层对上层只暴露了`Sync/Async/Oneway`三个接口，所以对于特定的Request/Response、连接的管理、路由等信息的处理和维护、
-其它`producer/consumer`共用的逻辑，均在这里实现。
-- 业务逻辑层：在这里实现各种producer、consumer的语义。所有producer、consumer专有的逻辑，均放到这里实现，如`queue selector`,
-`consume balance`, `offset storage`等。除了基础的数据结构外，producer和consumer之间内部的代码不能进行复用。
-
-
-## 设计目标
-0. 兼容cgo版本已经暴露出去的API
-1. 实现语义清晰、轻量的API接口
-2. 依赖关系清晰简单，设计和实现要正交
-
-## 目录结构
-### 源码
-- producer：producer相关逻辑
-- consumer：consumer相关逻辑
-- common（可改名）：连接管理和路由管理相关的通用逻辑
-- remote：网络通信和序列化
-
-### 其它
-- benchmark：压力测试相关代码
-- core：1.2版本cgo的代码库，该目录下的代码将会被移除，只保留API进行兼容，并会被标记为`Deprecated`
-- docs：文档，包括面向用户和开发者
-- examples：示例代码
-- test：集成测试代码
-
-## API
-
-### remote
-```go
-NewRemotingCommand(code int16, header CustomHeader) *RemotingCommand
-
-// send a request to servers and return until response received.
-SendMessageSync(ctx context.Context, brokerAddrs, brokerName string, request *SendMessageRequest, msgs []*Message) (*SendResult, error)
-
-SendMessageAsync(ctx context.Context, brokerAddrs, brokerName string, request *SendMessageRequest, msgs []*Message, f func(result *SendResult)) error
-
-SendMessageOneWay(ctx context.Context, brokerAddrs string, request *SendMessageRequest, msgs []*Message) (*SendResult, error)
-```
-
-### common
-All struct needed has been defined in codebase.
-
-```go
-// PullMessage with sync
-SendMessage(topic string, msgs *[]Message) error
-
-// SendMessageAsync send message with batch by async
-SendMessageAsync(topic string, msgs *[]Message, f func(result *SendResult)) error
-
-// PullMessage with sync
-PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequest) (*PullResult, error)
-
-// PullMessageAsync pull message async
-func PullMessageAsync(ctx context.Context, brokerAddrs string, request *PullMessageRequest, f func(result *PullResult)) error
-
-// QueryMaxOffset with specific queueId and topic
-QueryMaxOffset(topic string, queueId int) error
-
-// QueryConsumerOffset with specific queueId and topic of consumerGroup
-QueryConsumerOffset(consumerGroup, topic string, queue int) (int64, error)
-
-// SearchOffsetByTimestamp with specific queueId and topic
-SearchOffsetByTimestamp(topic string, queue int, timestamp int64) (int64, error)
-
-// UpdateConsumerOffset with specific queueId and topic
-UpdateConsumerOffset(consumerGroup, topic string, queue int, offset int64) error
-```
-
-## Road map
-for more details about features: [feature-list](../feature.md)
-
-### Milestone1(due: 2019.3.10)
-
-#### producer
-- [ ] normal message
-- [ ] order message
-
-#### consumer
-- [ ] normal message with pull/push
-- [ ] order message with pull/push
-- [ ] rebalance
-- [ ] offset manager
-
-#### common
-- [ ] API wrapper
-- [ ] connections manager
-- [ ] route
-
-#### remote
-- [ ] serializer
-- [ ] communication
-- [ ] processor
-- [ ] RPC
-
-### Milestone2 (2019.4.12)
-- Transaction Message
-- ACL
-- Message Tracing
-
-## sub project
-- RocketMQ Administration tools: JVM too heavy for command line tools
\ No newline at end of file
diff --git a/docs/zh/rocketmq-protocol_zh.md b/docs/zh/rocketmq-protocol_zh.md
deleted file mode 100644
index 818d2e1..0000000
--- a/docs/zh/rocketmq-protocol_zh.md
+++ /dev/null
@@ -1,117 +0,0 @@
-# RocketMQ 通信协议
-
-在 RocketMQ 中，`RemotingCommand` 是 RocketMQ 通信的基本对象，Request/Response 最后均被包装成 `RemotingCommand`。一个 `RemotingCommand` 在被序列化后的格式如下：
-
-```
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ frame_size | header_length | header_body | body +
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ 4bytes | 4bytes | (21 + r_len + e_len) bytes | remain bytes +
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-```
-
-| item | type | description |
-| :-: | :-: | :-: |
-| frame_size | `int32` | 一个 `RemotingCommand` 数据包大小 |
-| header_length | `int32` | 高8位表示数据的序列化方式，余下的表示真实 header 长度 |
-| header_body | `[]byte` | header 的 payload，长度由附带的 `remark` 和 `properties` 决定|
-| body | `[]byte` | 具体 Request/Response 的 payload |
-
-## Header
-
-RocketMQ 的 Header 序列化方式有两种：JSON 和 RocketMQ 私有的序列化方式。JSON 序列化方式不再赘述。具体可以参考 Java `RemotingCommand` 类。
-
-主要介绍 RocketMQ 的私有序列化方式。
-
-在序列化的时候，需要将序列化方式记录进数据包里面，即对 `header_length` 进行编码
-
-```go
-// 编码算法
-
-// 编码后的 header_length
-var header_length int32
-
-// 实际的 header 长度
-var headerDataLen int32
-
-// 序列化方式
-var SerializedType byte
-
-result := make([]byte, 4)
-result[0]|SerializedType
-result[1]|byte((headerDataLen >> 16) & 0xFF)
-result[2]|byte((headerDataLen >> 8) & 0xFF)
-result[3]|byte(headerDataLen & 0xFF)
-binary.Read(result, binary.BigEndian, &header_length)
-
-// 解码算法
-headerDataLen := header_length & 0xFFFFFF
-SerializedType := byte((header_length >> 24) & 0xFF)
-```
-
-### Header Frame
-
-```
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ request_code | l_flag | v_flag | opaque | request_flag | r_len | r_body | e_len | e_body +
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ 2bytes | 1byte | 2bytes | 4bytes | 4 bytes | 4 bytes | r_len bytes | 4 bytes | e_len bytes +
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-```
-
-| item | type | description |
-| :-: | :-: | :-: |
-| request_code | `int16` | 哪一种 Request 或 ResponseCode，具体类别由 request_flag 决定 |
-| l_flag | `byte` | language 位，用来标识Request来源方的开发语言 |
-| v_flag | `int16` | 版本标记位 |
-| request_flag |`int32`| Header标记位，用来标记该 `RemotingCommand` 的类型和请求方式 |
-| opaque | `int32` | 标识 Request/Response 的 RequestID，Broker 返回的 Response 通过该值和 Client 缓存的 Request 一一对应 |
-| r_len | `int32` | length of remark, remark 是 Request/Response 的附带说明信息，一般在 Response 中用来说明具体的错误原因 |
-| r_body | `[]byte` | payload of remark |
-| e_len | `int32` | length of extended fields，即 properties，一些非标准字段会存储在这里，在 RocketMQ 的各种 feature 中均有广泛应用 |
-| e_body | `int32` | payload of extended fields |
-
-## Body
-
-`body` 是具体的 Request/Response 的数据，在 RocketMQ 中，有许多种 Request/Response。每个类有自己的序列化和反序列方式，由于种类过多，
-这里就不再展开。可以具体参考Java代码中对`CommandCustomHeader`的使用。下面列一些 Client 使用到的 Request 和 Response。
-
-### RequestCode
-
-| item | type | description |
-| :-: | :-: | :-: |
-| SEND_MESSAGE | 10 | 向broker发送消息 |
-| PULL_MESSAGE | 11 | 从broker拉取消息，client的push模式也是通过pull的长轮询来实现的 |
-| TODO... | | |
-
-### ResponseCode
-
-| item | type | description |
-| :-: | :-: | :-: |
-| FLUSH_DISK_TIMEOUT | 10 | broker 存储层刷盘超时 |
-| SLAVE_NOT_AVAILABLE | 11 | slave 节点无法服务 |
-| FLUSH_SLAVE_TIMEOUT | 12 | 数据同步到 slave 超时 |
-| MESSAGE_ILLEGAL | 13 | 消息格式不合格 |
-| SERVICE_NOT_AVAILABLE | 14 | broker 暂时不可用 |
-| VERSION_NOT_SUPPORTED | 15 | 不支持的请求，目前没有看到使用 |
-| NO_PERMISSION | 16 | 对 broker、topic 或 subscription 无访问权限 |
-| TOPIC_EXIST_ALREADY | 18 | topic 已存在，目前没看到使用 |
-| PULL_NOT_FOUND | 19 | 没拉到消息，大多为 offset 错误 |
-| PULL_RETRY_IMMEDIATELY | 20 | 建议 client 立即重新拉取消息 |
-| PULL_OFFSET_MOVED | 21 | offset 太小或太大 |
-| QUERY_NOT_FOUND | 22 | 管理面 Response，TODO |
-| SUBSCRIPTION_PARSE_FAILED | 23 | 订阅数据解析失败 |
-| SUBSCRIPTION_NOT_EXIST | 24 | 订阅不存在 |
-| SUBSCRIPTION_NOT_LATEST | 25 | 订阅数据版本和 request 数据版本不匹配 |
-| SUBSCRIPTION_GROUP_NOT_EXIST | 26 | 订阅组不存在 |
-| FILTER_DATA_NOT_EXIST | 27 | filter 数据不存在 |
-| FILTER_DATA_NOT_LATEST | 28 | filter 数据版本和 request 数据版本不匹配 |
-| TRANSACTION_SHOULD_COMMIT | 200 | 事务 Response，TODO |
-| TRANSACTION_SHOULD_ROLLBACK | 201 | 事务 Response，TODO |
-| TRANSACTION_STATE_UNKNOW | 202 | 事务 Response，TODO | |
-| TRANSACTION_STATE_GROUP_WRONG | 203 | 事务 Response，TODO |
-| NO_BUYER_ID | 204 | 不知道是什么，没看到 broker 端在使用 |
-| NOT_IN_CURRENT_UNIT | 205 | 不知道是什么，没看到 broker 端在使用 |
-| CONSUMER_NOT_ONLINE | 206 | consumer 不在线，控制面 response |
-| CONSUME_MSG_TIMEOUT | 207 | client request 等待 broker 相应超时 |
-| NO_MESSAGE | 208 | 控制面 response，由 client 自己设置，不清楚具体用途 |
diff --git a/errors/errors.go b/errors/errors.go
deleted file mode 100644
index 195984e..0000000
--- a/errors/errors.go
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package errors
-
-import "errors"
-
-var (
- ErrRequestTimeout = errors.New("request timeout")
- ErrMQEmpty = errors.New("MessageQueue is nil")
- ErrOffset = errors.New("offset < 0")
- ErrNumbers = errors.New("numbers < 0")
- ErrEmptyTopic = errors.New("empty topic")
- ErrEmptyNameSrv = errors.New("empty namesrv")
- ErrEmptyGroupID = errors.New("empty group id")
- ErrTestMin = errors.New("test minutes must be positive integer")
- ErrOperationInterval = errors.New("operation interval must be positive integer")
- ErrMessageBody = errors.New("message body size must be positive integer")
- ErrEmptyExpression = errors.New("empty expression")
- ErrCreated = errors.New("consumer group has been created")
- ErrBrokerNotFound = errors.New("broker can not found")
- ErrStartTopic = errors.New("cannot subscribe topic since client either failed to start or has been shutdown.")
- ErrResponse = errors.New("response error")
- ErrCompressLevel = errors.New("unsupported compress level")
- ErrUnknownIP = errors.New("unknown IP address")
- ErrService = errors.New("service close is not running, please check")
- ErrTopicNotExist = errors.New("topic not exist")
- ErrNotExisted = errors.New("not existed")
- ErrNoNameserver = errors.New("nameServerAddrs can't be empty.")
- ErrMultiIP = errors.New("multiple IP addr does not support")
- ErrIllegalIP = errors.New("IP addr error")
- ErrTopicEmpty = errors.New("topic is nil")
- ErrMessageEmpty = errors.New("message is nil")
- ErrNotRunning = errors.New("producer not started")
- ErrPullConsumer = errors.New("pull consumer has not supported")
- ErrMultipleTopics = errors.New("the topic of the messages in one batch should be the same")
-)
diff --git a/examples/admin/topic/main.go b/examples/admin/topic/main.go
deleted file mode 100644
index ef9a536..0000000
--- a/examples/admin/topic/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
-
- "github.com/apache/rocketmq-client-go/v2/admin"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- topic := "newOne"
- //clusterName := "DefaultCluster"
- nameSrvAddr := []string{"127.0.0.1:9876"}
- brokerAddr := "127.0.0.1:10911"
-
- testAdmin, err := admin.NewAdmin(admin.WithResolver(primitive.NewPassthroughResolver(nameSrvAddr)))
- if err != nil {
- fmt.Println(err.Error())
- }
-
- //create topic
- err = testAdmin.CreateTopic(
- context.Background(),
- admin.WithTopicCreate(topic),
- admin.WithBrokerAddrCreate(brokerAddr),
- )
- if err != nil {
- fmt.Println("Create topic error:", err.Error())
- }
-
- //deletetopic
- err = testAdmin.DeleteTopic(
- context.Background(),
- admin.WithTopicDelete(topic),
- //admin.WithBrokerAddrDelete(brokerAddr),
- //admin.WithNameSrvAddr(nameSrvAddr),
- )
- if err != nil {
- fmt.Println("Delete topic error:", err.Error())
- }
-
- err = testAdmin.Close()
- if err != nil {
- fmt.Printf("Shutdown admin error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/acl/main.go b/examples/consumer/acl/main.go
deleted file mode 100644
index 96d0dd0..0000000
--- a/examples/consumer/acl/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- )
- if err != nil {
- fmt.Println("init consumer error: " + err.Error())
- os.Exit(0)
- }
-
- err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/broadcast/main.go b/examples/consumer/broadcast/main.go
deleted file mode 100644
index 219ab3e..0000000
--- a/examples/consumer/broadcast/main.go
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerModel(consumer.BroadCasting),
- )
- err := c.Subscribe("min", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/delay/main.go b/examples/consumer/delay/main.go
deleted file mode 100644
index 1bbe6ed..0000000
--- a/examples/consumer/delay/main.go
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
-
- for _, msg := range msgs {
- t := time.Now().UnixNano()/int64(time.Millisecond) - msg.BornTimestamp
- fmt.Printf("Receive message[msgId=%s] %d ms later\n", msg.MsgId, t)
- }
-
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/interceptor/main.go b/examples/consumer/interceptor/main.go
deleted file mode 100644
index 83f43b0..0000000
--- a/examples/consumer/interceptor/main.go
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithInterceptor(UserFistInterceptor(), UserSecondInterceptor()))
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
-
-func UserFistInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- msgCtx, _ := primitive.GetConsumerCtx(ctx)
- fmt.Printf("msgCtx: %v, mehtod: %s", msgCtx, primitive.GetMethod(ctx))
-
- msgs := req.([]*primitive.MessageExt)
- fmt.Printf("user first interceptor before invoke: %v\n", msgs)
- e := next(ctx, msgs, reply)
-
- holder := reply.(*consumer.ConsumeResultHolder)
- fmt.Printf("user first interceptor after invoke: %v, result: %v\n", msgs, holder)
- return e
- }
-}
-
-func UserSecondInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- msgs := req.([]*primitive.MessageExt)
- fmt.Printf("user second interceptor before invoke: %v\n", msgs)
- e := next(ctx, msgs, reply)
- holder := reply.(*consumer.ConsumeResultHolder)
- fmt.Printf("user second interceptor after invoke: %v, result: %v\n", msgs, holder)
- return e
- }
-}
diff --git a/examples/consumer/namespace/main.go b/examples/consumer/namespace/main.go
deleted file mode 100644
index d46f210..0000000
--- a/examples/consumer/namespace/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- consumer.WithNamespace("namespace"),
- )
- if err != nil {
- fmt.Println("init consumer error: " + err.Error())
- os.Exit(0)
- }
-
- err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/orderly/main.go b/examples/consumer/orderly/main.go
deleted file mode 100644
index 0f33e5f..0000000
--- a/examples/consumer/orderly/main.go
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerOrder(true),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
- fmt.Printf("orderly context: %v\n", orderlyCtx)
- fmt.Printf("subscribe orderly callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/pull/main.go b/examples/consumer/pull/main.go
deleted file mode 100644
index 5b5819e..0000000
--- a/examples/consumer/pull/main.go
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-func main() {
- c, err := rocketmq.NewPullConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- if err != nil {
- rlog.Fatal(fmt.Sprintf("fail to new pullConsumer: %s", err), nil)
- }
- err = c.Start()
- if err != nil {
- rlog.Fatal(fmt.Sprintf("fail to new pullConsumer: %s", err), nil)
- }
-
- ctx := context.Background()
- queue := primitive.MessageQueue{
- Topic: "TopicTest",
- BrokerName: "", // replace with your broker name. otherwise, pull will failed.
- QueueId: 0,
- }
-
- offset := int64(0)
- for {
- resp, err := c.PullFrom(ctx, queue, offset, 10)
- if err != nil {
- if err == errors.ErrRequestTimeout {
- fmt.Printf("timeout \n")
- time.Sleep(1 * time.Second)
- continue
- }
- fmt.Printf("unexpectable err: %v \n", err)
- return
- }
- if resp.Status == primitive.PullFound {
- fmt.Printf("pull message success. nextOffset: %d \n", resp.NextBeginOffset)
- for _, msg := range resp.GetMessageExts() {
- fmt.Printf("pull msg: %v \n", msg)
- }
- }
- offset = resp.NextBeginOffset
- }
-}
diff --git a/examples/consumer/retry/concurrent/main.go b/examples/consumer/retry/concurrent/main.go
deleted file mode 100644
index 49f16d1..0000000
--- a/examples/consumer/retry/concurrent/main.go
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// use concurrent consumer model, when Subscribe function return consumer.ConsumeRetryLater, the message will be
-// send to RocketMQ retry topic. we could set DelayLevelWhenNextConsume in ConsumeConcurrentlyContext, which used to
-// indicate the delay of message re-send to origin topic from retry topic.
-//
-// in this example, we always set DelayLevelWhenNextConsume=1, means that the message will be sent to origin topic after
-// 1s. in case of the unlimited retry, we will return consumer.ConsumeSuccess after ReconsumeTimes > 5
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- )
-
- // The DelayLevel specify the waiting time that before next reconsume,
- // and it range is from 1 to 18 now.
- //
- // The time of each level is the value of indexing of {level-1} in [1s, 5s, 10s, 30s,
- // 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 1h, 2h]
- delayLevel := 1
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback len: %d \n", len(msgs))
-
- concurrentCtx, _ := primitive.GetConcurrentlyCtx(ctx)
- concurrentCtx.DelayLevelWhenNextConsume = delayLevel // only run when return consumer.ConsumeRetryLater
-
- for _, msg := range msgs {
- if msg.ReconsumeTimes > 5 {
- fmt.Printf("msg ReconsumeTimes > 5. msg: %v", msg)
- return consumer.ConsumeSuccess, nil
- } else {
- fmt.Printf("subscribe callback: %v \n", msg)
- }
- }
- return consumer.ConsumeRetryLater, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shundown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/retry/order/main.go b/examples/consumer/retry/order/main.go
deleted file mode 100644
index e2d13bf..0000000
--- a/examples/consumer/retry/order/main.go
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * use orderly consumer model, when Subscribe function return consumer.SuspendCurrentQueueAMoment, it will be re-send to
- * local msg queue for later consume if msg.ReconsumeTimes < MaxReconsumeTimes, otherwise, it will be send to rocketmq
- * DLQ topic, we should manually resolve the msg.
- */
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerOrder(true),
- consumer.WithMaxReconsumeTimes(5),
- )
-
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
- fmt.Printf("orderly context: %v\n", orderlyCtx)
- fmt.Printf("subscribe orderly callback len: %d \n", len(msgs))
-
- for _, msg := range msgs {
- if msg.ReconsumeTimes > 5 {
- fmt.Printf("msg ReconsumeTimes > 5. msg: %v", msg)
- } else {
- fmt.Printf("subscribe orderly callback: %v \n", msg)
- }
- }
- return consumer.SuspendCurrentQueueAMoment, nil
-
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shundown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/simple/main.go b/examples/consumer/simple/main.go
deleted file mode 100644
index 70f35b3..0000000
--- a/examples/consumer/simple/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- for i := range msgs {
- fmt.Printf("subscribe callback: %v \n", msgs[i])
- }
-
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/strategy/main.go b/examples/consumer/strategy/main.go
deleted file mode 100644
index 269ce11..0000000
--- a/examples/consumer/strategy/main.go
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithStrategy(consumer.AllocateByAveragely),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/tag/main.go b/examples/consumer/tag/main.go
deleted file mode 100644
index ec16f51..0000000
--- a/examples/consumer/tag/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- selector := consumer.MessageSelector{
- Type: consumer.TAG,
- Expression: "TagA || TagC",
- }
- err := c.Subscribe("TopicTest", selector, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/trace/main.go b/examples/consumer/trace/main.go
deleted file mode 100644
index 97f22ed..0000000
--- a/examples/consumer/trace/main.go
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- namesrvs := []string{"127.0.0.1:9876"}
- traceCfg := &primitive.TraceConfig{
- Access: primitive.Local,
- Resolver: primitive.NewPassthroughResolver(namesrvs),
- }
-
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithTrace(traceCfg),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
-
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/acl/main.go b/examples/producer/acl/main.go
deleted file mode 100644
index 96881d3..0000000
--- a/examples/producer/acl/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, err := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- )
-
- if err != nil {
- fmt.Println("init producer error: " + err.Error())
- os.Exit(0)
- }
-
- err = p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 100000; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/async/main.go b/examples/producer/async/main.go
deleted file mode 100644
index 2a5182c..0000000
--- a/examples/producer/async/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "sync"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-// Package main implements a async producer to send message.
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithQueueSelector(producer.NewManualQueueSelector()))
-
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- var wg sync.WaitGroup
- for i := 0; i < 10; i++ {
- wg.Add(1)
- err := p.SendAsync(context.Background(),
- func(ctx context.Context, result *primitive.SendResult, e error) {
- if e != nil {
- fmt.Printf("receive message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", result.String())
- }
- wg.Done()
- }, primitive.NewMessage("test", []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- }
- }
- wg.Wait()
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/batch/main.go b/examples/producer/batch/main.go
deleted file mode 100644
index dc591a1..0000000
--- a/examples/producer/batch/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- var msgs []*primitive.Message
- for i := 0; i < 10; i++ {
- msgs = append(msgs, primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client! num: "+strconv.Itoa(i))))
- }
-
- res, err := p.SendSync(context.Background(), msgs...)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/delay/main.go b/examples/producer/delay/main.go
deleted file mode 100644
index aadbb70..0000000
--- a/examples/producer/delay/main.go
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 10; i++ {
- msg := primitive.NewMessage("test", []byte("Hello RocketMQ Go Client!"))
- msg.WithDelayTimeLevel(3)
- res, err := p.SendSync(context.Background(), msg)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/interceptor/main.go b/examples/producer/interceptor/main.go
deleted file mode 100644
index c40f9a0..0000000
--- a/examples/producer/interceptor/main.go
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithInterceptor(UserFirstInterceptor(), UserSecondInterceptor()),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 10; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
-
-func UserFirstInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- fmt.Printf("user first interceptor before invoke: req:%v\n", req)
- err := next(ctx, req, reply)
- fmt.Printf("user first interceptor after invoke: req: %v, reply: %v \n", req, reply)
- return err
- }
-}
-
-func UserSecondInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- fmt.Printf("user second interceptor before invoke: req: %v\n", req)
- err := next(ctx, req, reply)
- fmt.Printf("user second interceptor after invoke: req: %v, reply: %v \n", req, reply)
- return err
- }
-}
diff --git a/examples/producer/namespace/main.go b/examples/producer/namespace/main.go
deleted file mode 100644
index 9124284..0000000
--- a/examples/producer/namespace/main.go
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, err := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- producer.WithNamespace("namespace"),
- )
-
- if err != nil {
- fmt.Println("init producer error: " + err.Error())
- os.Exit(0)
- }
-
- err = p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 100000; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/simple/main.go b/examples/producer/simple/main.go
deleted file mode 100644
index 6fd3364..0000000
--- a/examples/producer/simple/main.go
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-// Package main implements a simple producer to send message.
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- topic := "test"
-
- for i := 0; i < 10; i++ {
- msg := &primitive.Message{
- Topic: topic,
- Body: []byte("Hello RocketMQ Go Client! " + strconv.Itoa(i)),
- }
- res, err := p.SendSync(context.Background(), msg)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/tag/main.go b/examples/producer/tag/main.go
deleted file mode 100644
index 7ce8559..0000000
--- a/examples/producer/tag/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- tags := []string{"TagA", "TagB", "TagC"}
- for i := 0; i < 3; i++ {
- tag := tags[i%3]
- msg := primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!"))
- msg.WithTag(tag)
-
- res, err := p.SendSync(context.Background(), msg)
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/trace/main.go b/examples/producer/trace/main.go
deleted file mode 100644
index b741704..0000000
--- a/examples/producer/trace/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- namesrvs := []string{"127.0.0.1:9876"}
- traceCfg := &primitive.TraceConfig{
- Access: primitive.Local,
- Resolver: primitive.NewPassthroughResolver(namesrvs),
- }
-
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithTrace(traceCfg))
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 1; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
-
- time.Sleep(10 * time.Second)
-
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/transaction/main.go b/examples/producer/transaction/main.go
deleted file mode 100644
index 05b6c52..0000000
--- a/examples/producer/transaction/main.go
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-type DemoListener struct {
- localTrans *sync.Map
- transactionIndex int32
-}
-
-func NewDemoListener() *DemoListener {
- return &DemoListener{
- localTrans: new(sync.Map),
- }
-}
-
-func (dl *DemoListener) ExecuteLocalTransaction(msg *primitive.Message) primitive.LocalTransactionState {
- nextIndex := atomic.AddInt32(&dl.transactionIndex, 1)
- fmt.Printf("nextIndex: %v for transactionID: %v\n", nextIndex, msg.TransactionId)
- status := nextIndex % 3
- dl.localTrans.Store(msg.TransactionId, primitive.LocalTransactionState(status+1))
-
- fmt.Printf("dl")
- return primitive.UnknowState
-}
-
-func (dl *DemoListener) CheckLocalTransaction(msg *primitive.MessageExt) primitive.LocalTransactionState {
- fmt.Printf("%v msg transactionID : %v\n", time.Now(), msg.TransactionId)
- v, existed := dl.localTrans.Load(msg.TransactionId)
- if !existed {
- fmt.Printf("unknow msg: %v, return Commit", msg)
- return primitive.CommitMessageState
- }
- state := v.(primitive.LocalTransactionState)
- switch state {
- case 1:
- fmt.Printf("checkLocalTransaction COMMIT_MESSAGE: %v\n", msg)
- return primitive.CommitMessageState
- case 2:
- fmt.Printf("checkLocalTransaction ROLLBACK_MESSAGE: %v\n", msg)
- return primitive.RollbackMessageState
- case 3:
- fmt.Printf("checkLocalTransaction unknow: %v\n", msg)
- return primitive.UnknowState
- default:
- fmt.Printf("checkLocalTransaction default COMMIT_MESSAGE: %v\n", msg)
- return primitive.CommitMessageState
- }
-}
-
-func main() {
- p, _ := rocketmq.NewTransactionProducer(
- NewDemoListener(),
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(1),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s\n", err.Error())
- os.Exit(1)
- }
-
- for i := 0; i < 10; i++ {
- res, err := p.SendMessageInTransaction(context.Background(),
- primitive.NewMessage("TopicTest5", []byte("Hello RocketMQ again "+strconv.Itoa(i))))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- time.Sleep(5 * time.Minute)
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/go.mod b/go.mod
deleted file mode 100644
index 0aac6ba..0000000
--- a/go.mod
+++ /dev/null
@@ -1,17 +0,0 @@
-module github.com/apache/rocketmq-client-go/v2
-
-go 1.13
-
-require (
- github.com/emirpasic/gods v1.12.0
- github.com/golang/mock v1.3.1
- github.com/json-iterator/go v1.1.9
- github.com/pkg/errors v0.8.1
- github.com/sirupsen/logrus v1.4.1
- github.com/smartystreets/goconvey v1.6.4
- github.com/stathat/consistent v1.0.0
- github.com/stretchr/testify v1.3.0
- github.com/tidwall/gjson v1.13.0
- go.uber.org/atomic v1.5.1
- stathat.com/c/consistent v1.0.0 // indirect
-)
diff --git a/go.sum b/go.sum
deleted file mode 100644
index 157f415..0000000
--- a/go.sum
+++ /dev/null
@@ -1,63 +0,0 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
-github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
-github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
-github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
-go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
-stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
diff --git a/internal/callback.go b/internal/callback.go
deleted file mode 100644
index fea11b4..0000000
--- a/internal/callback.go
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "net"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// remotingClient callback TransactionProducer
-type CheckTransactionStateCallback struct {
- Addr net.Addr
- Msg *primitive.MessageExt
- Header CheckTransactionStateRequestHeader
-}
diff --git a/internal/client.go b/internal/client.go
deleted file mode 100644
index c7f3e58..0000000
--- a/internal/client.go
+++ /dev/null
@@ -1,893 +0,0 @@
... 11153 lines suppressed ...