You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/06/01 01:54:58 UTC
[inlong] branch master updated: [INLONG-4961][DataProxy] Add dataproxy-sdk-golang (#8111)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 55520198f [INLONG-4961][DataProxy] Add dataproxy-sdk-golang (#8111)
55520198f is described below
commit 55520198f95adbb119bc88e41cf44a2005f9fea1
Author: gunli <24...@qq.com>
AuthorDate: Thu Jun 1 09:54:51 2023 +0800
[INLONG-4961][DataProxy] Add dataproxy-sdk-golang (#8111)
Co-authored-by: gunli <gu...@tencent.com>
---
.licenserc.yaml | 1 +
.../dataproxy-sdk-golang/.golangci.yml | 40 ++
.../dataproxy-sdk-golang/Makefile | 30 +
.../dataproxy-sdk-golang/README.md | 189 ++++++
.../dataproxy-sdk-golang/bufferpool/buffer_pool.go | 50 ++
.../dataproxy-sdk-golang/bytecloser/bytecloser.go | 73 ++
.../dataproxy-sdk-golang/connpool/connpool.go | 287 ++++++++
.../dataproxy-sdk-golang/dataproxy/client.go | 369 +++++++++++
.../dataproxy-sdk-golang/dataproxy/discoverer.go | 239 +++++++
.../dataproxy-sdk-golang/dataproxy/example_test.go | 90 +++
.../dataproxy-sdk-golang/dataproxy/message.go | 29 +
.../dataproxy-sdk-golang/dataproxy/metrics.go | 428 ++++++++++++
.../dataproxy-sdk-golang/dataproxy/options.go | 184 ++++++
.../dataproxy/options_basic.go | 184 ++++++
.../dataproxy/options_producer.go | 105 +++
.../dataproxy-sdk-golang/dataproxy/request.go | 347 ++++++++++
.../dataproxy-sdk-golang/dataproxy/worker.go | 734 +++++++++++++++++++++
.../dataproxy-sdk-golang/dataproxy/worker_test.go | 77 +++
.../dataproxy-sdk-golang/discoverer/discoverer.go | 88 +++
.../dataproxy-sdk-golang/discoverer/dns.go | 223 +++++++
.../dataproxy-sdk-golang/discoverer/dns_test.go | 42 ++
.../dataproxy-sdk-golang/framer/framer.go | 171 +++++
.../dataproxy-sdk-golang/go.mod | 51 ++
.../dataproxy-sdk-golang/go.sum | 556 ++++++++++++++++
.../dataproxy-sdk-golang/logger/logger.go | 95 +++
.../dataproxy-sdk-golang/syncx/semaphore.go | 124 ++++
.../dataproxy-sdk-golang/util/bytestring.go | 31 +
.../dataproxy-sdk-golang/util/hash.go | 27 +
.../dataproxy-sdk-golang/util/id.go | 59 ++
.../dataproxy-sdk-golang/util/ip.go | 109 +++
pom.xml | 1 +
31 files changed, 5033 insertions(+)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 927564f15..629ff90fa 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -91,5 +91,6 @@ header:
# Go sum
- '**/tubemq-client-go/go.sum'
+ - '**/dataproxy-sdk-golang/go.sum'
comment: on-failure
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/.golangci.yml b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/.golangci.yml
new file mode 100755
index 000000000..861f3fcfd
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/.golangci.yml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+run:
+ timeout: 10m
+ tests: false
+linters:
+ disable-all: true
+ enable:
+ - asciicheck
+ - bodyclose
+ - durationcheck
+ - errcheck
+ - exportloopref
+ - gofmt
+ - goimports
+ - gosimple
+ - ineffassign
+ - makezero
+ - prealloc
+ - predeclared
+ - rowserrcheck
+ - staticcheck
+ - stylecheck
+ - typecheck
+ - unused
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/Makefile b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/Makefile
new file mode 100755
index 000000000..8f7cc4aaf
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/Makefile
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+.PHONY : clean all lint
+
+all :
+ gofmt -w .
+ goimports -w .
+ go vet ./... || exit 1
+ golint -set_exit_status ./... || exit 2
+
+ go build dataproxy/*
+clean :
+
+lint:
+ golangci-lint run
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/README.md b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/README.md
new file mode 100755
index 000000000..27815affd
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/README.md
@@ -0,0 +1,189 @@
+## Overview
+
+dataproxy-sdk-golang is the golang version of InLong DataProxy client SDK.
+
+## Features
+
+- Service discovery;
+- Connection pool, buffer pool, byte pool;
+- Backoff retry;
+- Concurrently batch send;
+- Send synchronously;
+- Send asynchronously;
+- Close gracefully;
+- Hookable debug log;
+- Heartbeat;
+- Metrics;
+- Snappy compress;
+- Additional column;
+- Server offline re-balance;
+
+## Usage
+
+### example
+
+refer: cli/main.go
+
+``` go
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "log"
+ "strings"
+ "time"
+
+ "go.uber.org/atomic"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy"
+)
+
+var (
+ set string
+ url string
+ groupID string
+ streamID string
+ payload string
+ count int
+ addCols mapFlag
+ async bool
+ succeed atomic.Int32
+ failed atomic.Int32
+)
+
+type mapFlag map[string]string
+
+func (f mapFlag) String() string {
+ return fmt.Sprintf("%v", map[string]string(f))
+}
+
+func (f mapFlag) Set(value string) error {
+ split := strings.SplitN(value, "=", 2)
+ if len(split) < 2 {
+ return errors.New("invalid map flag")
+ }
+
+ f[split[0]] = split[1]
+ return nil
+}
+
+func main() {
+ addCols = make(map[string]string)
+ flag.StringVar(&set, "set", "SH_IEG", "dataproxy set")
+ flag.StringVar(&url, "url", dataproxy.DefaultURL, "dataproxy URL")
+ flag.StringVar(&groupID, "group-id", "b_ieg_tglogv3_test", "dataproxy group ID")
+ flag.StringVar(&streamID, "stream-id", "GameSvrState", "dataproxy stream ID")
+ flag.StringVar(&payload, "payload", "GameSvrState|GameSvrId-Test|2023-01-11 10:08:30|127.0.0.1|1", "message payload")
+ flag.IntVar(&count, "count", 10, "send count")
+ flag.Var(&addCols, "col", "add columns, for example: -col k1=v1 -col k2=v2")
+ flag.BoolVar(&async, "async", false, "send asynchronously")
+ flag.Parse()
+
+ var err error
+ client, err := dataproxy.NewClient(
+ dataproxy.WithSet(set),
+ dataproxy.WithGroupID(groupID),
+ dataproxy.WithURL(url),
+ dataproxy.WithMetricsName("clit"),
+ dataproxy.WithAddColumns(addCols),
+ )
+
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ msg := dataproxy.Message{GroupID: groupID, StreamID: streamID, Payload: []byte(payload)}
+ for i := 0; i < count; i++ {
+ if !async {
+ err = client.Send(context.Background(), msg)
+ if err != nil {
+ fmt.Println(err)
+ }
+ } else {
+ client.SendAsync(context.Background(), msg, onResult)
+ }
+ }
+
+ if async {
+ wait()
+ }
+}
+
+func onResult(msg dataproxy.Message, err error) {
+ if err != nil {
+ fmt.Println("error message, streamID = " + msg.StreamID + ", Payload = " + string(msg.Payload))
+ failed.Add(1)
+ } else {
+ succeed.Add(1)
+ }
+}
+
+func wait() {
+ for {
+ if int(succeed.Load()+failed.Load()) >= count {
+ fmt.Println("succeed:", succeed.Load())
+ fmt.Println("failed:", failed.Load())
+ return
+ }
+ time.Sleep(1 * time.Second)
+ }
+}
+
+```
+
+### Options
+
+refer: dataproxy/options.go
+
+``` go
+// Options is the DataProxy go client configs
+type Options struct {
+ Set string // the set name of the server
+ GroupID string // InLong group ID
+ URL string // URL where the discoverer to get the endpoint list of the server
+ UpdateInterval time.Duration // interval to refresh the endpoint list, default: 5m
+ ConnTimeout time.Duration // connection timeout: default: 3000ms
+ WriteBufferSize int // write buffer size in bytes, default: 16M
+ ReadBufferSize int // read buffer size in bytes, default: 16M
+ SocketSendBufferSize int // socket send buffer size in bytes, default: 16M
+ SocketRecvBufferSize int // socket receive buffer size in bytes, default: 16M
+ BufferPool bufferpool.BufferPool // encoding/decoding buffer pool, if not given, SDK will init a new one
+ BytePool bufferpool.BytePool // encoding/decoding byte pool, if not given, SDK will init a new one
+ BufferPoolSize int // buffer pool size, default: 409600
+ BytePoolSize int // byte pool size, default: 409600
+ BytePoolWidth int // byte pool width, default: equals to BatchingMaxSize
+ Logger logger.Logger // debug logger, default: stdout
+ MetricsName string // the unique metrics name of this SDK, used to isolate metrics in the case that more than 1 client are initialized in one process
+ MetricsRegistry prometheus.Registerer // metrics registry, default: prometheus.DefaultRegisterer
+ WorkerNum int // worker number, default: 8
+ SendTimeout time.Duration // send timeout, default: 20000ms
+ MaxRetries int // max retry count, default: 2
+ BatchingMaxPublishDelay time.Duration // the time period within which the messages sent will be batched, default: 10ms
+ BatchingMaxMessages int // the maximum number of messages permitted in a batch, default: 10
+ BatchingMaxSize int // the maximum number of bytes permitted in a batch, default: 4K
+ MaxPendingMessages int // the max size of the queue holding the messages pending to receive an acknowledgment from the broker, default: 409600
+ BlockIfQueueIsFull bool // whether Send and SendAsync block if producer's message queue is full, default: false
+ AddColumns map[string]string // addition columns to add to the message, for example: __addcol1__worldid=xxx&__addcol2__ip=yyy, all the message will be added 2 more columns with worldid=xxx and ip=yyy
+ addColumnStr string // the string format of the AddColumns, just a cache, used internal
+}
+```
+
+## FAQ
+
+Q: Why should I provide a MetricsName option?
+
+A: It is used to isolate the prometheus metrics in the case you initialize more than one client in a process.
+
+Q: What is the purpose of the "AddColumns" option?
+
+A: In some case, you may need to add some meta/headers to you message, AddColumns can help you to do that. AddColumns
+can add some fix columns and values to your message. For example: \_\_addcol1\_\_worldid=xxx&\_\_addcol2\_\_ip=yyy, all
+the messages will be updated with 2 more columns with worldid=xxx and ip=yyy.
+
+Q: How to hook the debug logger?
+
+A: The debug logger is defined as an interface, while logrus logger and zap sugar logger are compatible with that
+interface, so you can pass a logrus logger or zap sugar logger as the debug logger.
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bufferpool/buffer_pool.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bufferpool/buffer_pool.go
new file mode 100755
index 000000000..1d24c1d44
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bufferpool/buffer_pool.go
@@ -0,0 +1,50 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bufferpool
+
+import (
+ "bytes"
+
+ "github.com/oxtoacart/bpool"
+)
+
+// BufferPool buffer pool interface
+type BufferPool interface {
+ Get() *bytes.Buffer
+ Put(buffer *bytes.Buffer)
+}
+
+// BytePool byte pool interface
+type BytePool interface {
+ Get() []byte
+ Put(b []byte)
+}
+
+// NewSizedBuffer news a buffer pool with specific buffer size
+func NewSizedBuffer(poolSize, bufferSize int) BufferPool {
+ return bpool.NewSizedBufferPool(poolSize, bufferSize)
+}
+
+// NewBuffer news a buffer pool
+func NewBuffer(poolSize int) BufferPool {
+ return bpool.NewBufferPool(poolSize)
+}
+
+// NewBytePool news a byte pool
+func NewBytePool(poolSize, length int) BytePool {
+ return bpool.NewBytePool(poolSize, length)
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bytecloser/bytecloser.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bytecloser/bytecloser.go
new file mode 100755
index 000000000..4b83863a7
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bytecloser/bytecloser.go
@@ -0,0 +1,73 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bytecloser
+
+import (
+ "bytes"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bufferpool"
+)
+
+// ByteCloser is the interface of a byte Buffer that can be closed
+type ByteCloser interface {
+ // Bytes return the bytes
+ Bytes() []byte
+ // Close closes the bytes
+ Close()
+}
+
+// Bytes is the raw byte implementations of ByteCloser
+type Bytes []byte
+
+// Bytes return the bytes
+func (b Bytes) Bytes() []byte {
+ return b
+}
+
+// Close closes the bytes
+func (b Bytes) Close() {
+}
+
+// BufferBytes is the buffered byte implementations of ByteCloser
+type BufferBytes struct {
+ BufferPool bufferpool.BufferPool
+ Buffer *bytes.Buffer
+}
+
+// Bytes return the bytes
+func (b *BufferBytes) Bytes() []byte {
+ if b.Buffer == nil {
+ return nil
+ }
+
+ return b.Buffer.Bytes()
+}
+
+// Close closes the bytes
+func (b *BufferBytes) Close() {
+ if b.Buffer == nil {
+ return
+ }
+
+ if b.BufferPool == nil {
+ b.Buffer.Reset()
+ return
+ }
+
+ b.BufferPool.Put(b.Buffer)
+ b.Buffer = nil
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/connpool/connpool.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/connpool/connpool.go
new file mode 100755
index 000000000..ff3faa033
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/connpool/connpool.go
@@ -0,0 +1,287 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connpool
+
+import (
+ "context"
+ "errors"
+ "math"
+ "sync"
+ "time"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger"
+
+ "github.com/panjf2000/gnet/v2"
+ "go.uber.org/atomic"
+)
+
+// Dialer is the interface of a dialer that return a NetConn
+type Dialer interface {
+ Dial(addr string) (gnet.Conn, error)
+}
+
+// EndpointRestrictedConnPool is the interface of a simple endpoint restricted connection pool that
+// the connection's remote address must be in an endpoint list, if not, it will be closed and can
+// not be used anymore, it is useful for holding the connections to a service whose endpoints can
+// be changed at runtime.
+type EndpointRestrictedConnPool interface {
+ Get() (gnet.Conn, error)
+ Put(conn gnet.Conn, err error)
+ UpdateEndpoints(all, add, del []string)
+ NumPooled() int
+}
+
+// NewConnPool news a EndpointRestrictedConnPool
+func NewConnPool(initEndpoints []string, connsPerEndpoint, size int,
+ dialer Dialer, log logger.Logger) (EndpointRestrictedConnPool, error) {
+ if len(initEndpoints) == 0 {
+ return nil, errors.New("init endpoints is empty")
+ }
+
+ if connsPerEndpoint <= 0 {
+ connsPerEndpoint = 1
+ }
+
+ if dialer == nil {
+ return nil, errors.New("dialer is nil")
+ }
+
+ if log == nil {
+ return nil, errors.New("logger is nil")
+ }
+
+ initConnNum := len(initEndpoints) * connsPerEndpoint
+ if size <= 0 {
+ size = int(math.Max(1024, float64(initConnNum)))
+ }
+
+ // copy endpoints
+ endpoints := make([]string, 0, len(initEndpoints))
+ endpoints = append(endpoints, initEndpoints...)
+
+ pool := &connPool{
+ connChan: make(chan gnet.Conn, size),
+ endpoints: endpoints,
+ connsPerEndpoint: connsPerEndpoint,
+ dialer: dialer,
+ log: log,
+ }
+
+ // store endpoints to map
+ for _, e := range endpoints {
+ pool.endpointMap.Store(e, struct{}{})
+ }
+
+ err := pool.initConns(initConnNum)
+ if err != nil {
+ return nil, err
+ }
+
+ return pool, nil
+}
+
+type connPool struct {
+ sync.RWMutex
+ connChan chan gnet.Conn
+ index atomic.Uint64
+ endpoints []string
+ endpointMap sync.Map
+ connsPerEndpoint int
+ dialer Dialer
+ log logger.Logger
+}
+
+func (p *connPool) Get() (gnet.Conn, error) {
+ select {
+ case conn := <-p.connChan:
+ return conn, nil
+ default:
+ return p.newConn()
+ }
+}
+
+func (p *connPool) getEndpoint() string {
+ index := p.index.Load()
+ p.index.Add(1)
+
+ p.RLock()
+ ep := p.endpoints[index%uint64(len(p.endpoints))]
+ p.RUnlock()
+ return ep
+}
+
+func (p *connPool) newConn() (gnet.Conn, error) {
+ ep := p.getEndpoint()
+ return p.dialer.Dial(ep)
+}
+
+func (p *connPool) initConns(count int) error {
+ // create some conns and then put them back to the pool
+ conns := make([]gnet.Conn, 0)
+ for i := 0; i < count; i++ {
+ conn, err := p.newConn()
+ if err != nil {
+ return err
+ }
+
+ conns = append(conns, conn)
+ }
+
+ for _, conn := range conns {
+ p.Put(conn, nil)
+ }
+
+ return nil
+}
+
+func (p *connPool) Put(conn gnet.Conn, err error) {
+ if conn == nil {
+ return
+ }
+
+ remoteAddr := conn.RemoteAddr()
+ if remoteAddr == nil {
+ p.log.Error("conn is not nil but remote address is nil")
+ return
+ }
+
+ addr := remoteAddr.String()
+ _, ok := p.endpointMap.Load(addr)
+ if !ok {
+ p.log.Info("endpoint deleted, close its connection, addr:", addr)
+ CloseConn(conn, 2*time.Minute)
+ return
+ }
+
+ // if an error occurs, close it first and try to open a new one to make sure the balance of the traffic
+ if ok && err != nil {
+ p.log.Info("connection error, close it and try to open a new one, addr:", addr)
+ CloseConn(conn, 2*time.Minute)
+ newConn, err := p.dialer.Dial(addr)
+ if err != nil {
+ return
+ }
+
+ select {
+ case p.connChan <- newConn:
+ return
+ case <-time.After(1 * time.Second):
+ // connChan is full, close the new conn
+ newConn.Close()
+ return
+ }
+ }
+
+ select {
+ case p.connChan <- conn:
+ default:
+ // connChan is full, close the connection after 2m
+ CloseConn(conn, 2*time.Minute)
+ }
+}
+
+func (p *connPool) UpdateEndpoints(all, add, del []string) {
+ if len(all) == 0 {
+ return
+ }
+ p.log.Debug("UpdateEndpoints")
+ p.log.Debug("all:", all)
+ p.log.Debug("add:", add)
+ p.log.Debug("del:", del)
+ endpoints := make([]string, 0, len(all))
+ endpoints = append(endpoints, all...)
+ p.Lock()
+ p.endpoints = endpoints
+ p.Unlock()
+
+ // store new endpoints to map
+ p.log.Info("add new connections...")
+ for _, ep := range add {
+ p.endpointMap.Store(ep, struct{}{})
+
+ for i := 0; i < p.connsPerEndpoint; i++ {
+ conn, err := p.dialer.Dial(ep)
+ if err != nil {
+ p.log.Error("new connection failed, addr:", ep, ", err:", err)
+ continue
+ }
+
+ p.log.Info("endpoint added, open new connection, addr:", ep)
+ select {
+ case p.connChan <- conn:
+ continue
+ case <-time.After(1 * time.Second):
+ // connChan is full, close the new conn
+ conn.Close()
+ continue
+ }
+ }
+ }
+
+ //
+ delEndpoints := make(map[string]struct{})
+ for _, ep := range del {
+ p.endpointMap.Delete(ep)
+ delEndpoints[ep] = struct{}{}
+ }
+
+ if len(delEndpoints) == 0 {
+ return
+ }
+
+ // delete connections for deleted endpoints
+ p.log.Info("delete old connections...")
+ for i := 0; i < len(p.connChan); i++ {
+ conn, ok := <-p.connChan
+ if !ok {
+ break
+ }
+
+ addr := conn.RemoteAddr().String()
+ _, ok = delEndpoints[addr]
+ if ok {
+ p.log.Info("endpoint deleted, close its connection, addr:", addr)
+ CloseConn(conn, 2*time.Minute)
+ } else {
+ p.connChan <- conn
+ }
+ }
+}
+
+func (p *connPool) NumPooled() int {
+ return len(p.connChan)
+}
+
+// CloseConn closes a connection after a duration of time
+func CloseConn(conn gnet.Conn, after time.Duration) {
+ if after <= 0 {
+ conn.Close()
+ return
+ }
+
+ ctx := context.Background()
+ go func() {
+ select {
+ case <-time.After(after):
+ conn.Close()
+ return
+ case <-ctx.Done():
+ conn.Close()
+ return
+ }
+ }()
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/client.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/client.go
new file mode 100755
index 000000000..77d64c765
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/client.go
@@ -0,0 +1,369 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "math"
+ "sync"
+ "time"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/connpool"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/framer"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger"
+
+ "github.com/panjf2000/gnet/v2"
+ "go.uber.org/atomic"
+)
+
+// variables
+var (
+ ErrInvalidGroupID = errors.New("invalid group ID")
+ ErrInvalidURL = errors.New("invalid URL")
+ ErrNoEndpoint = errors.New("service has no endpoints")
+ ErrNoAvailableWorker = errors.New("no available worker")
+)
+
+// Client is the interface of a DataProxy client
+type Client interface {
+ // Send sends a message and wait for the result.
+ Send(ctx context.Context, msg Message) error
+ // SendAsync sends a message asynchronously, when the message is sent or timeout, the callback will be called.
+ SendAsync(ctx context.Context, msg Message, callback Callback)
+ // Close flushes all the message to the server and wait for the results or timeout, then close the producer.
+ Close()
+}
+
+type client struct {
+ *gnet.BuiltinEventEngine // inherited from the default event engin
+ options *Options // config options
+ discoverer discoverer.Discoverer // service discoverer
+ connPool connpool.EndpointRestrictedConnPool // connection pool
+ netClient *gnet.Client // client side network manager
+ workers []*worker // worker to do send and receive jobs
+ curWorkerIndex atomic.Uint64 // current worker index
+ log logger.Logger // debug logger
+ metrics *metrics // metrics
+ framer framer.Framer // response framer
+ closeOnce sync.Once //
+}
+
+// NewClient Creates a dataproxy-go client instance
+func NewClient(opts ...Option) (Client, error) {
+ // default v1 options
+ options := &Options{}
+ for _, o := range opts {
+ o(options)
+ }
+ err := options.ValidateAndSetDefault()
+ if err != nil {
+ return nil, err
+ }
+ // the client struct
+ cli := &client{
+ options: options,
+ log: options.Logger,
+ }
+ err = cli.initAll()
+ if err != nil {
+ cli.Close()
+ return nil, err
+ }
+ err = cli.netClient.Start()
+ if err != nil {
+ cli.Close()
+ return nil, err
+ }
+ return cli, nil
+}
+
+func (c *client) initAll() error {
+ // the following initialization order must not be changed。
+ err := c.initDiscoverer()
+ if err != nil {
+ return err
+ }
+ err = c.initNetClient()
+ if err != nil {
+ return err
+ }
+ err = c.initConns()
+ if err != nil {
+ return err
+ }
+ err = c.initFramer()
+ if err != nil {
+ return err
+ }
+ err = c.initMetrics()
+ if err != nil {
+ return err
+ }
+ err = c.initWorkers()
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (c *client) initDiscoverer() error {
+ dis, err := NewDiscoverer(c.options.URL, c.options.GroupID, c.options.UpdateInterval, c.options.Logger)
+ if err != nil {
+ return err
+ }
+ c.discoverer = dis
+ dis.AddEventHandler(c)
+ return nil
+}
+
+func (c *client) initNetClient() error {
+ netClient, err := gnet.NewClient(
+ c,
+ gnet.WithLogger(c.options.Logger),
+ gnet.WithWriteBufferCap(c.options.WriteBufferSize),
+ gnet.WithReadBufferCap(c.options.ReadBufferSize),
+ gnet.WithSocketSendBuffer(c.options.SocketSendBufferSize),
+ gnet.WithSocketRecvBuffer(c.options.SocketRecvBufferSize))
+ if err != nil {
+ return err
+ }
+ // save net client
+ c.netClient = netClient
+ return nil
+}
+
+func (c *client) initConns() error {
+ epList := c.discoverer.GetEndpoints()
+ epLen := len(epList)
+ if epLen == 0 {
+ return ErrNoEndpoint
+ }
+
+ endpoints := make([]string, epLen)
+ for i := 0; i < epLen; i++ {
+ endpoints[i] = epList[i].Addr
+ }
+
+ // maximum connection number per endpoint is 3
+ connsPerEndpoint := c.options.WorkerNum/epLen + 1
+ connsPerEndpoint = int(math.Min(3, float64(connsPerEndpoint)))
+
+ pool, err := connpool.NewConnPool(endpoints, connsPerEndpoint, 512, c, c.log)
+ if err != nil {
+ return err
+ }
+
+ c.connPool = pool
+ return nil
+}
+
+func (c *client) initFramer() error {
+ framer, err := framer.NewLengthField(framer.LengthFieldCfg{
+ MaxFrameLen: 64 * 1024,
+ FieldOffset: 0,
+ FieldLength: 4,
+ Adjustment: 0,
+ BytesToStrip: 0,
+ })
+ if err != nil {
+ return err
+ }
+ c.framer = framer
+ return nil
+}
+
+func (c *client) initMetrics() error {
+ m, err := newMetrics(c.options.MetricsName, c.options.MetricsRegistry)
+ if err != nil {
+ return err
+ }
+ c.metrics = m
+ return nil
+}
+
+func (c *client) initWorkers() error {
+ c.workers = make([]*worker, 0, c.options.WorkerNum)
+ for i := 0; i < c.options.WorkerNum; i++ {
+ w, err := c.createWorker(i)
+ if err != nil {
+ return err
+ }
+ c.workers = append(c.workers, w)
+ }
+ return nil
+}
+
+func (c *client) Dial(addr string) (gnet.Conn, error) {
+ return c.netClient.Dial("tcp", addr)
+}
+
+func (c *client) Send(ctx context.Context, msg Message) error {
+ worker := c.getWorker()
+ if worker == nil {
+ return ErrNoAvailableWorker
+ }
+ return worker.send(ctx, msg)
+}
+
+func (c *client) SendAsync(ctx context.Context, msg Message, cb Callback) {
+ worker := c.getWorker()
+ if worker == nil {
+ if cb != nil {
+ cb(msg, ErrNoAvailableWorker)
+ }
+ return
+ }
+
+ worker.sendAsync(ctx, msg, cb)
+}
+
+func (c *client) getWorker() *worker {
+ for i := 0; i < c.options.WorkerNum; i++ {
+ index := c.curWorkerIndex.Load()
+ w := c.workers[index%uint64(len(c.workers))]
+ c.curWorkerIndex.Add(1)
+
+ if w.available() {
+ return w
+ } else if i == c.options.WorkerNum-1 {
+ c.metrics.incError(errAllWorkerBusy.strCode)
+ return w
+ } else {
+ time.Sleep(1 * time.Millisecond)
+ continue
+ }
+ }
+ return nil
+}
+
+func (c *client) Close() {
+ c.closeOnce.Do(func() {
+ if c.discoverer != nil {
+ c.discoverer.Close()
+ }
+ for _, w := range c.workers {
+ w.close()
+ }
+ if c.netClient != nil {
+ _ = c.netClient.Stop()
+ }
+ })
+}
+
+func (c *client) createWorker(index int) (*worker, error) {
+ return newWorker(c, index, c.options)
+}
+
+func (c *client) getConn() (gnet.Conn, error) {
+ return c.connPool.Get()
+}
+
+func (c *client) putConn(conn gnet.Conn, err error) {
+ c.connPool.Put(conn, err)
+}
+
+func (c *client) OnBoot(e gnet.Engine) gnet.Action {
+ c.log.Info("client boot")
+ return gnet.None
+}
+
+func (c *client) OnShutdown(e gnet.Engine) {
+ c.log.Info("client shutdown")
+}
+
+func (c *client) OnOpen(conn gnet.Conn) ([]byte, gnet.Action) {
+ c.log.Debug("connection opened: ", conn.RemoteAddr())
+ return nil, gnet.None
+}
+
+func (c *client) OnClose(conn gnet.Conn, err error) gnet.Action {
+ c.log.Warn("connection closed: ", conn.RemoteAddr(), ", err: ", err)
+ if err != nil {
+ c.metrics.incError(errConnClosedByPeer.strCode)
+ }
+ return gnet.None
+}
+
+func (c *client) OnTraffic(conn gnet.Conn) (action gnet.Action) {
+ // c.log.Debug("response received")
+ for {
+ total := conn.InboundBuffered()
+ if total < heartbeatRspLen {
+ break
+ }
+
+ buf, _ := conn.Peek(total)
+ // if it is a heartbeat response, skip it and read the next package
+ if bytes.Equal(buf[:heartbeatRspLen], heartbeatRsp) {
+ _, err := conn.Discard(heartbeatRspLen)
+ if err != nil {
+ c.metrics.incError(errConnReadFailed.getStrCode())
+ c.log.Error("discard connection stream failed, err", err)
+ // read failed, close the connection
+ return gnet.Close
+ }
+
+ c.log.Debug("heartbeat rsp receive")
+ continue
+ }
+
+ length, payloadOffset, payloadOffsetEnd, err := c.framer.ReadFrame(buf)
+ if err == framer.ErrIncompleteFrame {
+ break
+ }
+
+ if err != nil {
+ c.metrics.incError(errConnReadFailed.getStrCode())
+ c.log.Error("invalid packet from stream connection, close it, err:", err)
+ // read failed, close the connection
+ return gnet.Close
+ }
+
+ frame, _ := conn.Peek(length)
+ _, err = conn.Discard(length)
+ if err != nil {
+ c.metrics.incError(errConnReadFailed.getStrCode())
+ c.log.Error("discard connection stream failed, err", err)
+ // read failed, close the connection
+ return gnet.Close
+ }
+
+ // handle response
+ c.onResponse(frame[payloadOffset:payloadOffsetEnd])
+ }
+ return gnet.None
+}
+
+func (c *client) onResponse(frame []byte) {
+ rsp := batchRsp{}
+ rsp.decode(frame)
+
+ index := getWorkerIndex(rsp.workerID)
+ if index < 0 || index >= len(c.workers) {
+ c.log.Errorf("invalid worker index from response, index=%d", index)
+ return
+ }
+
+ c.workers[index].onRsp(rsp)
+}
+
+func (c *client) OnEndpointUpdate(all, add, del discoverer.EndpointList) {
+ c.connPool.UpdateEndpoints(all.Addresses(), add.Addresses(), del.Addresses())
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/discoverer.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/discoverer.go
new file mode 100755
index 000000000..3b6b76d1f
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/discoverer.go
@@ -0,0 +1,239 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "crypto/tls"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger"
+
+ "github.com/go-resty/resty/v2"
+)
+
+// NewDiscoverer news a DataProxy discoverer
+func NewDiscoverer(url, groupID string, lookupInterval time.Duration, log logger.Logger) (discoverer.Discoverer, error) {
+ if url == "" {
+ return nil, errors.New("URL is not given")
+ }
+ if groupID == "" {
+ return nil, errors.New("group ID is not given")
+ }
+ if lookupInterval <= 0 {
+ lookupInterval = 5 * time.Minute
+ }
+ if log == nil {
+ log = logger.Std()
+ }
+
+ dis := &dataProxyDiscoverer{
+ url: url,
+ groupID: groupID,
+ lookupInterval: lookupInterval,
+ endpointList: make([]discoverer.Endpoint, 0),
+ endpointListMap: make(map[string]discoverer.Endpoint),
+ eventHandlers: make(map[discoverer.EventHandler]struct{}),
+ log: log,
+ }
+
+ // initial lookup
+ dis.lookup()
+ dis.update()
+ return dis, nil
+}
+
+type dataProxyDiscoverer struct {
+ sync.RWMutex
+ url string
+ groupID string
+ lookupInterval time.Duration
+ endpointList discoverer.EndpointList
+ endpointListStr string
+ endpointListMap map[string]discoverer.Endpoint
+ eventHandlers map[discoverer.EventHandler]struct{}
+ closeFunc func()
+ log logger.Logger
+}
+
+func (d *dataProxyDiscoverer) GetEndpoints() discoverer.EndpointList {
+ d.RLock()
+ defer d.RUnlock()
+ return d.endpointList
+}
+
+func (d *dataProxyDiscoverer) AddEventHandler(h discoverer.EventHandler) {
+ d.Lock()
+ defer d.Unlock()
+ d.eventHandlers[h] = struct{}{}
+}
+
+func (d *dataProxyDiscoverer) DelEventHandler(h discoverer.EventHandler) {
+ d.Lock()
+ defer d.Unlock()
+ delete(d.eventHandlers, h)
+}
+
+func (d *dataProxyDiscoverer) Close() {
+ if d.closeFunc != nil {
+ d.closeFunc()
+ }
+}
+
+func (d *dataProxyDiscoverer) lookup() {
+ c, err := d.get(2)
+ if err != nil {
+ d.log.Error("get server endpoint list failed:", err)
+ return
+ }
+
+ if len(c.NodeList) == 0 {
+ d.log.Warn("server endpoint list is empty")
+ return
+ }
+
+ allEndpointAddrs := make([]string, 0, len(c.NodeList))
+ allEndpointAddrMap := make(map[string]discoverer.Endpoint)
+ allEndpoints := make([]discoverer.Endpoint, 0, len(c.NodeList))
+ addEndpoints := make([]discoverer.Endpoint, 0)
+ for _, ep := range c.NodeList {
+ addr := discoverer.BuildAddr(ep.IP, ep.Port)
+ newEndpoint := discoverer.Endpoint{Host: ep.IP, Port: ep.Port, Addr: addr}
+ allEndpointAddrs = append(allEndpointAddrs, addr)
+ allEndpointAddrMap[addr] = newEndpoint
+ allEndpoints = append(allEndpoints, newEndpoint)
+
+ // new endpoint is not found in the old endpoint list map, it is a new added one
+ if _, ok := d.endpointListMap[addr]; !ok {
+ addEndpoints = append(addEndpoints, newEndpoint)
+ }
+ }
+
+ delEndpoints := make([]discoverer.Endpoint, 0)
+ for addr, oldEndpoint := range d.endpointListMap {
+ // old endpoint is not found in the new endpoint list map, it is deleted
+ if _, ok := allEndpointAddrMap[addr]; !ok {
+ delEndpoints = append(delEndpoints, oldEndpoint)
+ }
+ }
+
+ d.Lock()
+ d.endpointList = allEndpoints
+ d.endpointListMap = allEndpointAddrMap
+ d.Unlock()
+ // show this logger only in case of ip list change
+ // only ticker will update d.allEndpointAddrStr, lock is unnecessary
+ sort.Strings(allEndpointAddrs)
+ allEndpointAddrStr := strings.Join(allEndpointAddrs, ";")
+ if allEndpointAddrStr != d.endpointListStr {
+ d.endpointListStr = allEndpointAddrStr
+ d.log.Infof("update server endpoint list %s: %v", d.url, allEndpointAddrStr)
+ }
+
+ d.RLock()
+ defer d.RUnlock()
+ if len(addEndpoints) > 0 || len(delEndpoints) > 0 {
+ for h := range d.eventHandlers {
+ h.OnEndpointUpdate(allEndpoints, addEndpoints, delEndpoints)
+ }
+ }
+}
+
+func (d *dataProxyDiscoverer) update() {
+ wg := sync.WaitGroup{}
+ ticker := time.NewTicker(d.lookupInterval)
+ stopCh := make(chan struct{})
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case <-ticker.C:
+ d.lookup()
+ case <-stopCh:
+ return
+ default:
+ time.Sleep(500 * time.Millisecond)
+ }
+ }
+ }()
+
+ d.closeFunc = func() {
+ ticker.Stop()
+ close(stopCh)
+ wg.Wait()
+ }
+}
+
+// get gets endpoint list from DataProxy service registry
+func (d *dataProxyDiscoverer) get(retry int) (*cluster, error) {
+ reqURL := fmt.Sprintf("%s/%s?protocolType=tcp", d.url, d.groupID)
+ client := resty.New().SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+
+ httpRsp, err := client.R().Post(reqURL)
+ if err != nil {
+ d.log.Error("get server endpoint list failed:", err)
+ if retry <= 1 {
+ return nil, err
+ }
+
+ retry--
+ return d.get(retry)
+ }
+
+ rsp := &response{}
+ err = json.Unmarshal(httpRsp.Body(), rsp)
+ if err != nil {
+ return nil, err
+ }
+
+ if !rsp.Success {
+ return nil, errors.New(rsp.ErrMsg)
+ }
+
+ return &rsp.Data, nil
+}
+
+// endpoint is the config of a cluster endpoint
+type endpoint struct {
+ ID int `json:"id"`
+ IP string `json:"ip"`
+ Port int `json:"port"`
+ NodeLoad int `json:"nodeLoad"`
+ ProtocolType string `json:"protocolType"`
+}
+
+// cluster is the config of a DataProxy cluster
+type cluster struct {
+ ClusterID int `json:"clusterId"`
+ Load int `json:"load"`
+ IsIntranet int `json:"isIntranet"`
+ IsSwitch int `json:"isSwitch"`
+ NodeList []endpoint `json:"nodeList"`
+}
+
+type response struct {
+ Success bool `json:"success"`
+ ErrMsg string `json:"errMsg"`
+ Data cluster `json:"data"`
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/example_test.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/example_test.go
new file mode 100644
index 000000000..112999571
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/example_test.go
@@ -0,0 +1,90 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy_test
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy"
+
+ "go.uber.org/atomic"
+)
+
+func ExampleClient_Send() {
+ client, err := dataproxy.NewClient(
+ dataproxy.WithGroupID("test"),
+ dataproxy.WithURL("http://127.0.0.1:8083/inlong/manager/openapi/dataproxy/getIpList"),
+ dataproxy.WithMetricsName("test"),
+ )
+
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+
+ for i := 0; i < 1000; i++ {
+ err := client.Send(context.Background(), dataproxy.Message{
+ GroupID: "test",
+ StreamID: "test",
+ Payload: []byte("test|a|b|c"),
+ })
+ if err != nil {
+ fmt.Println(err)
+ }
+ }
+
+ client.Close()
+}
+
+func ExampleClient_SendAsync() {
+ client, err := dataproxy.NewClient(
+ dataproxy.WithGroupID("test"),
+ dataproxy.WithURL("http://127.0.0.1:8083/inlong/manager/openapi/dataproxy/getIpList"),
+ dataproxy.WithMetricsName("test"),
+ )
+
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+
+ var success atomic.Uint64
+ var failed atomic.Uint64
+ for i := 0; i < 1000; i++ {
+ client.SendAsync(context.Background(),
+ dataproxy.Message{
+ GroupID: "test",
+ StreamID: "test",
+ Payload: []byte("test|a|b|c"),
+ },
+ func(msg dataproxy.Message, err error) {
+ if err != nil {
+ success.Add(1)
+ } else {
+ failed.Add(1)
+ }
+ })
+ }
+
+ // wait async send finish
+ time.Sleep(3 * time.Second)
+ fmt.Println("success:", success.Load())
+ fmt.Println("failed:", failed.Load())
+ client.Close()
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/message.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/message.go
new file mode 100755
index 000000000..717544b9e
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/message.go
@@ -0,0 +1,29 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+// Callback is the callback function signature of the DataProxy producer
+type Callback func(message Message, err error)
+
+// Message is the message to send
+type Message struct {
+ GroupID string // InLong group ID
+ StreamID string // InLong stream ID
+ Payload []byte // the content of the message
+ Headers map[string]string // message headers, won't be sent to the server right now
+ MetaData interface{} // any data you want, won't be sent to the server, but you can get it in the callback
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/metrics.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/metrics.go
new file mode 100755
index 000000000..218f87b96
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/metrics.go
@@ -0,0 +1,428 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "errors"
+ "sync"
+
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+type metrics struct {
+ errorCounter *prometheus.CounterVec
+ errorCounters map[string]prometheus.Counter
+ errorCounterLock sync.RWMutex
+ retryCounter *prometheus.CounterVec
+ retryCounters map[string]prometheus.Counter
+ retryCounterLock sync.RWMutex
+ timeoutCounter *prometheus.CounterVec
+ timeoutCounters map[string]prometheus.Counter
+ timeoutCounterLock sync.RWMutex
+ messageCounter *prometheus.CounterVec
+ messageCounters map[string]prometheus.Counter
+ messageCounterLock sync.RWMutex
+ updateConnCounter *prometheus.CounterVec
+ updateConnCounters map[string]prometheus.Counter
+ updateConnCounterLock sync.RWMutex
+ pendingMessageGauge *prometheus.GaugeVec
+ pendingMessageGauges map[string]prometheus.Gauge
+ pendingMessageGaugeLock sync.RWMutex
+ batchSizeHistogram *prometheus.HistogramVec
+ batchSizeHistograms map[string]prometheus.Observer
+ batchSizeHistogramLock sync.RWMutex
+ batchTimeHistogram *prometheus.HistogramVec
+ batchTimeHistograms map[string]prometheus.Observer
+ batchTimeHistogramLock sync.RWMutex
+ name string
+ registry prometheus.Registerer
+}
+
+func newMetrics(name string, reg prometheus.Registerer) (*metrics, error) {
+ if name == "" {
+ return nil, errors.New("metrics name is not given")
+ }
+
+ registry := prometheus.DefaultRegisterer
+ if reg != nil {
+ registry = reg
+ }
+
+ m := &metrics{
+ errorCounters: make(map[string]prometheus.Counter),
+ retryCounters: make(map[string]prometheus.Counter),
+ timeoutCounters: make(map[string]prometheus.Counter),
+ messageCounters: make(map[string]prometheus.Counter),
+ updateConnCounters: make(map[string]prometheus.Counter),
+ pendingMessageGauges: make(map[string]prometheus.Gauge),
+ batchSizeHistograms: make(map[string]prometheus.Observer),
+ batchTimeHistograms: make(map[string]prometheus.Observer),
+ name: name,
+ registry: registry,
+ }
+
+ err := m.init()
+ if err != nil {
+ return nil, err
+ }
+
+ return m, nil
+}
+
+func (m *metrics) init() error {
+ m.errorCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "data_proxy_error_count",
+ Help: "Counter of error events",
+ }, []string{"name", "code"})
+ err := m.registry.Register(m.errorCounter)
+ if err != nil {
+ are, ok := err.(prometheus.AlreadyRegisteredError)
+ if !ok {
+ return err
+ }
+ m.errorCounter = are.ExistingCollector.(*prometheus.CounterVec)
+ }
+
+ m.retryCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "data_proxy_retry_count",
+ Help: "Counter of retry batches",
+ }, []string{"name", "worker"})
+ err = m.registry.Register(m.retryCounter)
+ if err != nil {
+ are, ok := err.(prometheus.AlreadyRegisteredError)
+ if !ok {
+ return err
+ }
+ m.retryCounter = are.ExistingCollector.(*prometheus.CounterVec)
+ }
+
+ m.timeoutCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "data_proxy_timeout_count",
+ Help: "Counter of timeout batches",
+ }, []string{"name", "worker"})
+ err = m.registry.Register(m.timeoutCounter)
+ if err != nil {
+ are, ok := err.(prometheus.AlreadyRegisteredError)
+ if !ok {
+ return err
+ }
+ m.timeoutCounter = are.ExistingCollector.(*prometheus.CounterVec)
+ }
+
+ m.messageCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "data_proxy_msg_count",
+ Help: "Counter of message",
+ }, []string{"name", "code"})
+ err = m.registry.Register(m.messageCounter)
+ if err != nil {
+ are, ok := err.(prometheus.AlreadyRegisteredError)
+ if !ok {
+ return err
+ }
+ m.messageCounter = are.ExistingCollector.(*prometheus.CounterVec)
+ }
+
+ m.updateConnCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "data_proxy_update_conn_count",
+ Help: "Counter of update connection events",
+ }, []string{"name", "code"})
+ err = m.registry.Register(m.updateConnCounter)
+ if err != nil {
+ are, ok := err.(prometheus.AlreadyRegisteredError)
+ if !ok {
+ return err
+ }
+ m.updateConnCounter = are.ExistingCollector.(*prometheus.CounterVec)
+ }
+
+ m.pendingMessageGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "data_proxy_pending_msg_gauge",
+ Help: "Gauge of pending message",
+ }, []string{"name", "worker"})
+ err = m.registry.Register(m.pendingMessageGauge)
+ if err != nil {
+ are, ok := err.(prometheus.AlreadyRegisteredError)
+ if !ok {
+ return err
+ }
+ m.pendingMessageGauge = are.ExistingCollector.(*prometheus.GaugeVec)
+ }
+
+ m.batchSizeHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+ Name: "data_proxy_batch_size",
+ Help: "Histogram of batch size",
+ Buckets: []float64{1024, 2 * 1024, 4 * 1024, 8 * 1024, 16 * 1024, 32 * 1024, 64 * 1024, 128 * 1024, 256 * 1024},
+ }, []string{"name", "code"})
+ err = m.registry.Register(m.batchSizeHistogram)
+ if err != nil {
+ are, ok := err.(prometheus.AlreadyRegisteredError)
+ if !ok {
+ return err
+ }
+ m.batchSizeHistogram = are.ExistingCollector.(*prometheus.HistogramVec)
+ }
+
+ m.batchTimeHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+ Name: "data_proxy_batch_time",
+ Help: "Histogram of batch time in milliseconds",
+ Buckets: []float64{5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000},
+ }, []string{"name", "code"})
+ err = m.registry.Register(m.batchTimeHistogram)
+ if err != nil {
+ are, ok := err.(prometheus.AlreadyRegisteredError)
+ if !ok {
+ return err
+ }
+ m.batchTimeHistogram = are.ExistingCollector.(*prometheus.HistogramVec)
+ }
+
+ return nil
+}
+
+func (m *metrics) incError(code string) {
+ m.errorCounterLock.RLock()
+ c, ok := m.errorCounters[code]
+ m.errorCounterLock.RUnlock()
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ m.errorCounterLock.Lock()
+ defer m.errorCounterLock.Unlock()
+ c, ok = m.errorCounters[code]
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ c, err := m.errorCounter.GetMetricWithLabelValues(m.name, code)
+ if err != nil {
+ return
+ }
+
+ c.Add(float64(1))
+ m.errorCounters[code] = c
+}
+
+func (m *metrics) incRetry(worker string) {
+ m.retryCounterLock.RLock()
+ c, ok := m.retryCounters[worker]
+ m.retryCounterLock.RUnlock()
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ m.retryCounterLock.Lock()
+ defer m.retryCounterLock.Unlock()
+ c, ok = m.retryCounters[worker]
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ c, err := m.retryCounter.GetMetricWithLabelValues(m.name, worker)
+ if err != nil {
+ return
+ }
+
+ c.Add(float64(1))
+ m.retryCounters[worker] = c
+}
+
+func (m *metrics) incTimeout(worker string) {
+ m.timeoutCounterLock.RLock()
+ c, ok := m.timeoutCounters[worker]
+ m.timeoutCounterLock.RUnlock()
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ m.timeoutCounterLock.Lock()
+ defer m.timeoutCounterLock.Unlock()
+ c, ok = m.timeoutCounters[worker]
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ c, err := m.timeoutCounter.GetMetricWithLabelValues(m.name, worker)
+ if err != nil {
+ return
+ }
+
+ c.Add(float64(1))
+ m.timeoutCounters[worker] = c
+}
+
+func (m *metrics) incMessage(code string) {
+ m.messageCounterLock.RLock()
+ c, ok := m.messageCounters[code]
+ m.messageCounterLock.RUnlock()
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ m.messageCounterLock.Lock()
+ defer m.messageCounterLock.Unlock()
+ c, ok = m.messageCounters[code]
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ c, err := m.messageCounter.GetMetricWithLabelValues(m.name, code)
+ if err != nil {
+ return
+ }
+
+ c.Add(float64(1))
+ m.messageCounters[code] = c
+}
+
+func (m *metrics) incUpdateConn(code string) {
+ m.updateConnCounterLock.RLock()
+ c, ok := m.updateConnCounters[code]
+ m.updateConnCounterLock.RUnlock()
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ m.updateConnCounterLock.Lock()
+ defer m.updateConnCounterLock.Unlock()
+ c, ok = m.updateConnCounters[code]
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ c, err := m.updateConnCounter.GetMetricWithLabelValues(m.name, code)
+ if err != nil {
+ return
+ }
+
+ c.Add(float64(1))
+ m.updateConnCounters[code] = c
+}
+
+func (m *metrics) incPending(worker string) {
+ m.pendingMessageGaugeLock.RLock()
+ c, ok := m.pendingMessageGauges[worker]
+ m.pendingMessageGaugeLock.RUnlock()
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ m.pendingMessageGaugeLock.Lock()
+ defer m.pendingMessageGaugeLock.Unlock()
+ c, ok = m.pendingMessageGauges[worker]
+ if ok {
+ c.Add(float64(1))
+ return
+ }
+
+ c, err := m.pendingMessageGauge.GetMetricWithLabelValues(m.name, worker)
+ if err != nil {
+ return
+ }
+
+ c.Add(float64(1))
+ m.pendingMessageGauges[worker] = c
+}
+
+func (m *metrics) decPending(worker string) {
+ m.pendingMessageGaugeLock.RLock()
+ c, ok := m.pendingMessageGauges[worker]
+ m.pendingMessageGaugeLock.RUnlock()
+ if ok {
+ c.Dec()
+ return
+ }
+
+ m.pendingMessageGaugeLock.Lock()
+ defer m.pendingMessageGaugeLock.Unlock()
+ c, ok = m.pendingMessageGauges[worker]
+ if ok {
+ c.Dec()
+ return
+ }
+
+ c, err := m.pendingMessageGauge.GetMetricWithLabelValues(m.name, worker)
+ if err != nil {
+ return
+ }
+
+ c.Dec()
+ m.pendingMessageGauges[worker] = c
+}
+
+func (m *metrics) observeSize(code string, value int) {
+ m.batchSizeHistogramLock.RLock()
+ o, ok := m.batchSizeHistograms[code]
+ m.batchSizeHistogramLock.RUnlock()
+ if ok {
+ o.Observe(float64(value))
+ return
+ }
+
+ m.batchSizeHistogramLock.Lock()
+ defer m.batchSizeHistogramLock.Unlock()
+ o, ok = m.batchSizeHistograms[code]
+ if ok {
+ o.Observe(float64(value))
+ return
+ }
+
+ o, err := m.batchSizeHistogram.GetMetricWithLabelValues(m.name, code)
+ if err != nil {
+ return
+ }
+
+ o.Observe(float64(value))
+ m.batchSizeHistograms[code] = o
+}
+
+func (m *metrics) observeTime(code string, value int64) {
+ m.batchTimeHistogramLock.RLock()
+ o, ok := m.batchTimeHistograms[code]
+ m.batchTimeHistogramLock.RUnlock()
+ if ok {
+ o.Observe(float64(value))
+ return
+ }
+
+ m.batchTimeHistogramLock.Lock()
+ defer m.batchTimeHistogramLock.Unlock()
+ o, ok = m.batchTimeHistograms[code]
+ if ok {
+ o.Observe(float64(value))
+ return
+ }
+
+ o, err := m.batchTimeHistogram.GetMetricWithLabelValues(m.name, code)
+ if err != nil {
+ return
+ }
+
+ o.Observe(float64(value))
+ m.batchTimeHistograms[code] = o
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options.go
new file mode 100755
index 000000000..f506a915f
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options.go
@@ -0,0 +1,184 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "math"
+ "strings"
+ "time"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bufferpool"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util"
+
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+ ipPlaceholder = "{{ip}}"
+)
+
+var (
+ // DefaultURL is the default URL of the DataProxy service registry from where a discoverer can get DataProxy endpoints
+ DefaultURL = "http://127.0.0.1:8083/inlong/manager/openapi/dataproxy/getIpList"
+ localIP, _ = util.GetFirstPrivateIP()
+)
+
+// Options is the DataProxy go client configs
+type Options struct {
+ Set string // the set name of the server
+ GroupID string // InLong group ID
+ URL string // URL where the discoverer to get the endpoint list of the server
+ UpdateInterval time.Duration // interval to refresh the endpoint list, default: 5m
+ ConnTimeout time.Duration // connection timeout: default: 3000ms
+ WriteBufferSize int // write buffer size in bytes, default: 16M
+ ReadBufferSize int // read buffer size in bytes, default: 16M
+ SocketSendBufferSize int // socket send buffer size in bytes, default: 16M
+ SocketRecvBufferSize int // socket receive buffer size in bytes, default: 16M
+ BufferPool bufferpool.BufferPool // encoding/decoding buffer pool, if not given, SDK will init a new one
+ BytePool bufferpool.BytePool // encoding/decoding byte pool, if not given, SDK will init a new one
+ BufferPoolSize int // buffer pool size, default: 409600
+ BytePoolSize int // byte pool size, default: 409600
+ BytePoolWidth int // byte pool width, default: equals to BatchingMaxSize
+ Logger logger.Logger // debug logger, default: stdout
+ MetricsName string // the unique metrics name of this SDK, used to isolate metrics in the case that more than 1 client are initialized in one process
+ MetricsRegistry prometheus.Registerer // metrics registry, default: prometheus.DefaultRegisterer
+ WorkerNum int // worker number, default: 8
+ SendTimeout time.Duration // send timeout, default: 30000ms
+ MaxRetries int // max retry count, default: 2
+ BatchingMaxPublishDelay time.Duration // the time period within which the messages sent will be batched, default: 10ms
+ BatchingMaxMessages int // the maximum number of messages permitted in a batch, default: 10
+ BatchingMaxSize int // the maximum number of bytes permitted in a batch, default: 4K
+ MaxPendingMessages int // the max size of the queue holding the messages pending to receive an acknowledgment from the broker, default: 409600
+ BlockIfQueueIsFull bool // whether Send and SendAsync block if producer's message queue is full, default: false
+ AddColumns map[string]string // addition columns to add to the message, for example: __addcol1__worldid=xxx&__addcol2__ip=yyy, all the message will be added 2 more columns with worldid=xxx and ip=yyy
+ addColumnStr string // the string format of the AddColumns, just a cache, used internal
+}
+
+// ValidateAndSetDefault validates an options and set up the default values
+func (options *Options) ValidateAndSetDefault() error {
+ if options.GroupID == "" {
+ // 未指定服务器端口
+ return ErrInvalidGroupID
+ }
+
+ if options.URL == "" {
+ // 未指定服务器域名
+ return ErrInvalidURL
+ }
+
+ if options.UpdateInterval == 0 {
+ options.UpdateInterval = 5 * time.Minute
+ }
+
+ if options.ConnTimeout <= 0 {
+ options.ConnTimeout = 3 * time.Second
+ }
+
+ if options.BatchingMaxPublishDelay <= 0 {
+ options.BatchingMaxPublishDelay = 10 * time.Millisecond
+ }
+
+ if options.BatchingMaxMessages <= 0 {
+ options.BatchingMaxMessages = 10
+ }
+
+ if options.BatchingMaxSize <= 0 {
+ options.BatchingMaxSize = 4 * 1024
+ }
+
+ if options.MaxPendingMessages <= 0 {
+ options.MaxPendingMessages = 409600
+ }
+
+ if options.BufferPoolSize <= 0 {
+ options.BufferPoolSize = 409600
+ }
+
+ if options.BytePoolSize <= 0 {
+ options.BytePoolSize = 409600
+ }
+
+ if options.BytePoolWidth <= 0 {
+ options.BytePoolWidth = int(math.Max(4096, float64(options.BatchingMaxSize)))
+ }
+
+ if options.BufferPool == nil {
+ options.BufferPool = bufferpool.NewBuffer(options.BufferPoolSize)
+ }
+
+ if options.BytePool == nil {
+ options.BytePool = bufferpool.NewBytePool(options.BytePoolSize, options.BytePoolWidth)
+ }
+
+ if options.Logger == nil {
+ options.Logger = logger.Std()
+ }
+
+ if options.MetricsName == "" {
+ options.MetricsName = "dataproxy-go"
+ }
+
+ if options.MetricsRegistry == nil {
+ options.MetricsRegistry = prometheus.DefaultRegisterer
+ }
+
+ if options.WorkerNum <= 0 {
+ options.WorkerNum = 8
+ }
+
+ if options.SendTimeout <= 0 {
+ options.SendTimeout = 30 * time.Second
+ }
+
+ if options.MaxRetries <= 0 {
+ options.MaxRetries = 2
+ }
+
+ if options.WriteBufferSize <= 0 {
+ options.WriteBufferSize = 16 * 1024 * 1024
+ }
+
+ if options.ReadBufferSize <= 0 {
+ options.ReadBufferSize = 16 * 1024 * 1024
+ }
+
+ if options.SocketSendBufferSize <= 0 {
+ options.SocketSendBufferSize = 16 * 1024 * 1024
+ }
+
+ if options.SocketRecvBufferSize <= 0 {
+ options.SocketRecvBufferSize = 16 * 1024 * 1024
+ }
+
+ sb := strings.Builder{}
+ for k, v := range options.AddColumns {
+ sb.WriteString("&")
+ sb.WriteString(k)
+ sb.WriteString("=")
+ if v == ipPlaceholder {
+ // if the value is the IP address placeholder ,replace it with the local IP address
+ sb.WriteString(localIP)
+ } else {
+ sb.WriteString(v)
+ }
+
+ }
+ options.addColumnStr = sb.String()
+
+ return nil
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options_basic.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options_basic.go
new file mode 100755
index 000000000..552bbdf18
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options_basic.go
@@ -0,0 +1,184 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "time"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bufferpool"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger"
+
+ "github.com/prometheus/client_golang/prometheus"
+)
+
+// Option is the Options helper.
+type Option func(*Options)
+
+// WithSet sets Set
+func WithSet(s string) Option {
+ return func(o *Options) {
+ if s == "" {
+ return
+ }
+ o.Set = s
+ }
+}
+
+// WithGroupID sets GroupID
+func WithGroupID(g string) Option {
+ return func(o *Options) {
+ o.GroupID = g
+ }
+}
+
+// WithURL sets URL
+func WithURL(u string) Option {
+ return func(o *Options) {
+ o.URL = u
+ }
+}
+
+// WithUpdateInterval sets UpdateInterval
+func WithUpdateInterval(u time.Duration) Option {
+ return func(o *Options) {
+ o.UpdateInterval = u
+ }
+}
+
+// WithConnTimeout sets ConnTimeout
+func WithConnTimeout(t time.Duration) Option {
+ return func(o *Options) {
+ if t <= 0 {
+ return
+ }
+ o.ConnTimeout = t
+ }
+}
+
+// WithWriteBufferSize sets WriteBufferSize
+func WithWriteBufferSize(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.WriteBufferSize = n
+ }
+}
+
+// WithReadBufferSize sets ReadBufferSize
+func WithReadBufferSize(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.ReadBufferSize = n
+ }
+}
+
+// WithSocketSendBufferSize sets SocketSendBufferSize
+func WithSocketSendBufferSize(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.SocketSendBufferSize = n
+ }
+}
+
+// WithSocketRecvBufferSize sets SocketRecvBufferSize
+func WithSocketRecvBufferSize(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.SocketRecvBufferSize = n
+ }
+}
+
+// WithBufferPool sets BufferPool
+func WithBufferPool(bp bufferpool.BufferPool) Option {
+ return func(o *Options) {
+ o.BufferPool = bp
+ }
+}
+
+// WithBytePool sets BytePool
+func WithBytePool(bp bufferpool.BytePool) Option {
+ return func(o *Options) {
+ o.BytePool = bp
+ }
+}
+
+// WithBufferPoolSize sets BufferPoolSize
+func WithBufferPoolSize(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.BufferPoolSize = n
+ }
+}
+
+// WithBytePoolSize sets BytePoolSize
+func WithBytePoolSize(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.BytePoolSize = n
+ }
+}
+
+// WithBytePoolWidth sets BytePoolWidth
+func WithBytePoolWidth(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.BytePoolWidth = n
+ }
+}
+
+// WithLogger sets Logger
+func WithLogger(log logger.Logger) Option {
+ return func(o *Options) {
+ if log == nil {
+ return
+ }
+ o.Logger = log
+ }
+}
+
+// WithMetricsName sets Logger
+func WithMetricsName(name string) Option {
+ return func(o *Options) {
+ if name == "" {
+ return
+ }
+ o.MetricsName = name
+ }
+}
+
+// WithMetricsRegistry sets Logger
+func WithMetricsRegistry(reg prometheus.Registerer) Option {
+ return func(o *Options) {
+ if reg == nil {
+ return
+ }
+ o.MetricsRegistry = reg
+ }
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options_producer.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options_producer.go
new file mode 100755
index 000000000..088a5263b
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/options_producer.go
@@ -0,0 +1,105 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "time"
+)
+
+// WithWorkerNum sets WorkerNum
+func WithWorkerNum(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.WorkerNum = n
+ }
+}
+
+// WithSendTimeout sets SendTimeout
+func WithSendTimeout(t time.Duration) Option {
+ return func(o *Options) {
+ if t <= 0 {
+ return
+ }
+ o.SendTimeout = t
+ }
+}
+
+// WithMaxRetries sets MaxRetries
+func WithMaxRetries(n int) Option {
+ return func(o *Options) {
+ if n < 0 {
+ return
+ }
+ o.MaxRetries = n
+ }
+}
+
+// WithBatchingMaxPublishDelay sets BatchingMaxPublishDelay
+func WithBatchingMaxPublishDelay(t time.Duration) Option {
+ return func(o *Options) {
+ if t <= 0 {
+ return
+ }
+ o.BatchingMaxPublishDelay = t
+ }
+}
+
+// WithBatchingMaxMessages sets BatchingMaxMessages
+func WithBatchingMaxMessages(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.BatchingMaxMessages = n
+ }
+}
+
+// WithBatchingMaxSize sets BatchingMaxSize
+func WithBatchingMaxSize(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.BatchingMaxSize = n
+ }
+}
+
+// WithMaxPendingMessages sets MaxPendingMessages
+func WithMaxPendingMessages(n int) Option {
+ return func(o *Options) {
+ if n <= 0 {
+ return
+ }
+ o.MaxPendingMessages = n
+ }
+}
+
+// WithBlockIfQueueIsFull sets BlockIfQueueIsFull
+func WithBlockIfQueueIsFull(b bool) Option {
+ return func(o *Options) {
+ o.BlockIfQueueIsFull = b
+ }
+}
+
+// WithAddColumns sets AddColumns
+func WithAddColumns(cols map[string]string) Option {
+ return func(o *Options) {
+ o.AddColumns = cols
+ }
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/request.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/request.go
new file mode 100755
index 000000000..f45204c80
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/request.go
@@ -0,0 +1,347 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "strconv"
+ "strings"
+ "time"
+ "unsafe"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bufferpool"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/syncx"
+
+ "github.com/klauspost/compress/snappy"
+)
+
+var (
+ byteOrder = binary.BigEndian
+ heartbeatRsp = []byte{0x00, 0x00, 0x00, 0x01, 0x01}
+ heartbeatRspLen = len(heartbeatRsp)
+)
+
+const (
+ msgTypeBatch uint8 = 5
+ msgTypeHeartbeat uint8 = 1
+)
+
+type heartbeatReq struct {
+}
+
+func (h heartbeatReq) encode(buffer *bytes.Buffer) []byte {
+ var bodyLen uint32 = 0
+ var attrLen uint32 = 0
+ totalLen := 1 + 4 + bodyLen + 4 + attrLen
+
+ buffer.Grow(int(totalLen + 4))
+ uint32LenBuf := make([]byte, 4)
+ byteOrder.PutUint32(uint32LenBuf, totalLen)
+ _, _ = buffer.Write(uint32LenBuf)
+
+ _, _ = buffer.Write([]byte{msgTypeHeartbeat})
+
+ byteOrder.PutUint32(uint32LenBuf, bodyLen)
+ _, _ = buffer.Write(uint32LenBuf)
+
+ byteOrder.PutUint32(uint32LenBuf, attrLen)
+ _, _ = buffer.Write(uint32LenBuf)
+
+ return buffer.Bytes()
+}
+
+type batchCallback func()
+type batchReq struct {
+ workerID string
+ batchID string
+ groupID string
+ streamID string
+ dataReqs []*sendDataReq
+ dataSize int
+ batchTime time.Time
+ lastSendTime time.Time
+ retries int
+ bufferPool bufferpool.BufferPool
+ bytePool bufferpool.BytePool
+ buffer *bytes.Buffer
+ callback batchCallback
+ metrics *metrics
+ addColumns string
+}
+
+func (b *batchReq) append(req *sendDataReq) {
+ b.dataReqs = append(b.dataReqs, req)
+ b.dataSize += len(req.msg.Payload)
+}
+
+func (b *batchReq) done(err error) {
+ errorCode := getErrorCode(err)
+ for _, req := range b.dataReqs {
+ req.done(err, errorCode)
+ }
+
+ if b.callback != nil {
+ b.callback()
+ }
+
+ if b.buffer != nil && b.bufferPool != nil {
+ b.bufferPool.Put(b.buffer)
+ b.buffer = nil
+ }
+
+ if b.metrics != nil {
+ if errorCode != errOK.strCode {
+ b.metrics.incError(errorCode)
+ }
+ b.metrics.observeTime(errorCode, time.Since(b.batchTime).Milliseconds())
+ b.metrics.observeSize(errorCode, b.dataSize)
+ }
+}
+
+func (b *batchReq) encode() []byte {
+ if b.bufferPool == nil {
+ panic("batch req buffer pool is not set")
+ }
+
+ if b.bytePool == nil {
+ panic("batch req byte pool is not set")
+ }
+
+ if b.buffer != nil {
+ return b.buffer.Bytes()
+ }
+
+ b.buffer = b.bufferPool.Get()
+ bodyBuffer := b.bufferPool.Get()
+ defer b.bufferPool.Put(bodyBuffer)
+
+ uint32LenBuf := make([]byte, 4)
+ for _, req := range b.dataReqs {
+ // fix:no line feed at the end of the payload, or it will be a NULL record in the data warehouse
+ payload := req.msg.Payload
+ payloadLen := len(payload)
+ if payload[payloadLen-1] == '\n' {
+ payload = payload[0 : payloadLen-1]
+ payloadLen = len(payload)
+ }
+ dataLen := uint32(payloadLen)
+ // write body len, 4 bytes
+ byteOrder.PutUint32(uint32LenBuf, dataLen)
+ _, _ = bodyBuffer.Write(uint32LenBuf)
+ // write one body payload
+ _, _ = bodyBuffer.Write(payload)
+ }
+
+ // uncompressed body
+ bodyLen := bodyBuffer.Len()
+ body := bodyBuffer.Bytes()
+
+ // compress it when the body length is > 256
+ compress := bodyLen > 256
+ var compressBuf []byte
+ if compress {
+ compressBuf = b.bytePool.Get()
+ defer b.bytePool.Put(compressBuf)
+
+ requiredCompressBufSize := snappy.MaxEncodedLen(bodyLen)
+ if requiredCompressBufSize <= 0 {
+ compress = false
+ } else {
+ if cap(compressBuf) < requiredCompressBufSize {
+ compressBuf = make([]byte, requiredCompressBufSize)
+ }
+ body = snappy.Encode(compressBuf, body)
+ bodyLen = len(body)
+ }
+ }
+
+ sb := strings.Builder{}
+ sb.Grow(256)
+ sb.WriteString("groupId=")
+ sb.WriteString(b.groupID)
+
+ sb.WriteString("&streamId=")
+ sb.WriteString(b.streamID)
+
+ sb.WriteString("&dt=")
+ sb.WriteString(strconv.FormatInt(time.Now().UnixMilli(), 10))
+
+ sb.WriteString("&mid=")
+ sb.WriteString(b.batchID)
+
+ // use batchID as sid, the server use sid to de-duping a batch
+ sb.WriteString("&sid=")
+ sb.WriteString(b.batchID)
+
+ sb.WriteString("&wid=")
+ sb.WriteString(b.workerID)
+
+ sb.WriteString("&cnt=")
+ sb.WriteString(strconv.FormatInt(int64(len(b.dataReqs)), 10))
+
+ // if compressed, add one more attribute
+ if compress {
+ sb.WriteString("&cp=snappy")
+ }
+
+ // add columns
+ if b.addColumns != "" {
+ sb.WriteString(b.addColumns)
+ }
+
+ attr := sb.String()
+ attrLen := sb.Len()
+
+ // msg type: 1 byte, body length: 4 bytes, body, attributes length: 4 byte, attributes
+ totalLen := 1 + 4 + bodyLen + 4 + attrLen
+
+ // add more 4 bytes to store the total length
+ b.buffer.Grow(int(totalLen + 4))
+ // total length: 4 bytes
+ byteOrder.PutUint32(uint32LenBuf, uint32(totalLen))
+ _, _ = b.buffer.Write(uint32LenBuf)
+
+ // msg type: 1 byte
+ _, _ = b.buffer.Write([]byte{msgTypeBatch})
+
+ // body length: 4 bytes
+ byteOrder.PutUint32(uint32LenBuf, uint32(bodyLen))
+ _, _ = b.buffer.Write(uint32LenBuf)
+ // body
+ _, _ = b.buffer.Write(body)
+
+ // attributes length: 4 byte
+ byteOrder.PutUint32(uint32LenBuf, uint32(attrLen))
+ _, _ = b.buffer.Write(uint32LenBuf)
+ // attributes
+ _, _ = b.buffer.Write(*(*[]byte)(unsafe.Pointer(&attr)))
+
+ return b.buffer.Bytes()
+}
+
+type batchRsp struct {
+ errCode int
+ workerID string
+ batchID string
+ groupID string
+ streamID string
+ dt string
+}
+
+func (b *batchRsp) decode(input []byte) {
+ // var totalLen uint32
+ // var msgType uint8
+ var bodyLen uint32
+ var attrLen uint32
+
+ // buffer := bytes.NewBuffer(input)
+ // _ = binary.Read(buffer, byteOrder, &totalLen)
+ // totalLen = byteOrder.Uint32(input[0:4])
+
+ // _ = binary.Read(buffer, byteOrder, &msgType)
+ // msgType = input[4]
+
+ // _ = binary.Read(buffer, byteOrder, &bodyLen)
+ bodyLen = byteOrder.Uint32(input[5:9])
+
+ // body := make([]byte, bodyLen)
+ // _ = binary.Read(buffer, byteOrder, body)
+ // body := input[9 : 9+bodyLen]
+
+ // _ = binary.Read(buffer, byteOrder, &attrLen)
+ attrLen = byteOrder.Uint32(input[9+bodyLen : 9+bodyLen+4])
+ // attr := make([]byte, attrLen)
+ // _ = binary.Read(buffer, byteOrder, attr)
+ attr := input[9+bodyLen+4 : 9+bodyLen+4+attrLen]
+
+ // fmt.Println(string(attr))
+
+ attrList := strings.Split(string(attr), "&")
+ for _, item := range attrList {
+ kv := strings.Split(item, "=")
+ if len(kv) != 2 {
+ continue
+ }
+
+ k := kv[0]
+ v := kv[1]
+ switch k {
+ case "errCode":
+ b.errCode, _ = strconv.Atoi(v)
+ case "groupId":
+ b.groupID = v
+ case "mid":
+ b.batchID = v
+ case "streamId":
+ b.streamID = v
+ case "wid":
+ b.workerID = v
+ case "dt":
+ b.dt = v
+ }
+ }
+}
+
+type sendDataReq struct {
+ ctx context.Context
+ msg Message
+ callback Callback
+ flushImmediately bool
+ publishTime time.Time
+ semaphore syncx.Semaphore
+ metrics *metrics
+ workerID string
+}
+
+func (s *sendDataReq) done(err error, errCode string) {
+ if s.semaphore != nil {
+ s.semaphore.Release()
+ }
+
+ if s.callback != nil {
+ s.callback(s.msg, err)
+ }
+
+ if s.metrics != nil {
+ if s.semaphore != nil {
+ s.metrics.decPending(s.workerID)
+ }
+ if errCode == "" {
+ errCode = getErrorCode(err)
+ }
+
+ s.metrics.incMessage(errCode)
+ }
+}
+
+type closeReq struct {
+ doneCh chan struct{}
+}
+
+func getWorkerIndex(workerID string) int {
+ if workerID == "" {
+ return -1
+ }
+
+ index, err := strconv.Atoi(workerID)
+ if err != nil {
+ return -1
+ }
+ return index
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/worker.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/worker.go
new file mode 100755
index 000000000..0734c06c5
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/worker.go
@@ -0,0 +1,734 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "context"
+ "math/rand"
+ "runtime/debug"
+ "strconv"
+ "time"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/bufferpool"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/syncx"
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util"
+
+ "github.com/panjf2000/gnet/v2"
+ "go.uber.org/atomic"
+)
+
+const (
+ defaultHeartbeatInterval = 60
+ defaultMapCleanInterval = 20
+ defaultMapCleanThreshold = 500000
+)
+
+type workerState int32
+
+const (
+ // worker states
+ stateInit = iota
+ stateReady
+ stateClosing
+ stateClosed
+)
+
+var (
+ errOK = &errNo{code: 0, strCode: "0", message: "OK"}
+ errSendTimeout = &errNo{code: 10001, strCode: "10001", message: "message send timeout"}
+ errSendFailed = &errNo{code: 10002, strCode: "10002", message: "message send failed"} //nolint:unused
+ errProducerClosed = &errNo{code: 10003, strCode: "10003", message: "producer already been closed"}
+ errSendQueueIsFull = &errNo{code: 10004, strCode: "10004", message: "producer send queue is full"}
+ errContextExpired = &errNo{code: 10005, strCode: "10005", message: "message context expired"}
+ errNewConnFailed = &errNo{code: 10006, strCode: "10006", message: "new conn failed"}
+ errConnWriteFailed = &errNo{code: 10007, strCode: "10007", message: "conn write failed"}
+ errConnReadFailed = &errNo{code: 10008, strCode: "10008", message: "conn read failed"}
+ errLogToLong = &errNo{code: 10009, strCode: "10009", message: "input log is too long"} //nolint:unused
+ errBadLog = &errNo{code: 10010, strCode: "10010", message: "input log is invalid"}
+ errServerError = &errNo{code: 10011, strCode: "10011", message: "server error"}
+ errServerPanic = &errNo{code: 10012, strCode: "10012", message: "server panic"}
+ errAllWorkerBusy = &errNo{code: 10013, strCode: "10013", message: "all workers are busy"}
+ errNoMatchReq4Rsp = &errNo{code: 10014, strCode: "10014", message: "no match unacknowledged request for response"}
+ errConnClosedByPeer = &errNo{code: 10015, strCode: "10015", message: "conn closed by peer"}
+ errUnknown = &errNo{code: 20001, strCode: "20001", message: "unknown"}
+)
+
+type errNo struct {
+ code int
+ strCode string
+ message string
+}
+
+func (e *errNo) Error() string {
+ return e.message
+}
+
+//nolint:unused
+func (e *errNo) getCode() int {
+ return e.code
+}
+
+func (e *errNo) getStrCode() string {
+ return e.strCode
+}
+
+func getErrorCode(err error) string {
+ if err == nil {
+ return errOK.getStrCode()
+ }
+
+ switch t := err.(type) {
+ case *errNo:
+ return t.getStrCode()
+ default:
+ return errUnknown.getStrCode()
+ }
+}
+
+type worker struct {
+ client *client // parent client
+ index int // worker id
+ indexStr string // worker id string
+ options *Options // config options
+ state atomic.Int32 // worker state
+ log logger.Logger // debug logger
+ conn atomic.Value // connection used to send data
+ cmdChan chan interface{} // command channel
+ dataChan chan *sendDataReq // data channel
+ dataSemaphore syncx.Semaphore // semaphore used to handle message queueing
+ pendingBatches map[string]*batchReq // pending batches
+ unackedBatches map[string]*batchReq // sent but not acknowledged batches
+ sendFailedBatches chan *batchReq // send failed batches channel
+ retryBatches chan *batchReq // retry batches channel
+ responseBatches chan batchRsp // batch response channel
+ batchTimeoutTicker *time.Ticker // batch timeout ticker
+ sendTimeoutTicker *time.Ticker // send timeout ticker
+ heartbeatTicker *time.Ticker // heartbeat ticker
+ mapCleanTicker *time.Ticker // map clean ticker, clean the unackedBatches map periodically
+ updateConnTicker *time.Ticker // update connection ticker, change connection periodically
+ unackedBatchCount int // sent but not acknowledged batches counter, used to clean the unackedBatches map periodically
+ metrics *metrics // metrics
+ bufferPool bufferpool.BufferPool // buffer pool
+ bytePool bufferpool.BytePool // byte pool
+ stop bool // stop the worker
+}
+
+func newWorker(cli *client, index int, opts *Options) (*worker, error) {
+ sendTimeout := opts.SendTimeout / 2
+ if sendTimeout == 0 {
+ sendTimeout = 5 * time.Second
+ }
+
+ w := &worker{
+ index: index,
+ indexStr: strconv.Itoa(index),
+ client: cli,
+ options: opts,
+ cmdChan: make(chan interface{}),
+ dataChan: make(chan *sendDataReq, opts.MaxPendingMessages),
+ dataSemaphore: syncx.NewSemaphore(int32(opts.MaxPendingMessages)),
+ pendingBatches: make(map[string]*batchReq),
+ unackedBatches: make(map[string]*batchReq),
+ sendFailedBatches: make(chan *batchReq, opts.MaxPendingMessages),
+ retryBatches: make(chan *batchReq, opts.MaxPendingMessages),
+ responseBatches: make(chan batchRsp, opts.MaxPendingMessages),
+ batchTimeoutTicker: time.NewTicker(opts.BatchingMaxPublishDelay),
+ sendTimeoutTicker: time.NewTicker(sendTimeout),
+ heartbeatTicker: time.NewTicker(defaultHeartbeatInterval * time.Second),
+ mapCleanTicker: time.NewTicker(defaultMapCleanInterval * time.Second),
+ updateConnTicker: time.NewTicker(time.Duration(30+rand.Intn(50)) * time.Second), // update connection randomly
+ metrics: cli.metrics,
+ bufferPool: opts.BufferPool,
+ bytePool: opts.BytePool,
+ log: opts.Logger,
+ }
+
+ // set to init state
+ w.setState(stateInit)
+
+ // get a connection
+ conn, err := cli.getConn()
+ if err != nil {
+ return nil, err
+ }
+ w.setConn(conn)
+
+ // start the worker
+ w.start()
+ // update to ready state
+ w.setState(stateReady)
+
+ return w, nil
+}
+
+func (w *worker) available() bool {
+ return w.dataSemaphore.Available() > 0
+}
+
+func (w *worker) start() {
+ go func() {
+ defer func() {
+ if rec := recover(); rec != nil {
+ w.log.Errorf("panic:", rec)
+ w.log.Error(string(debug.Stack()))
+ w.metrics.incError(errServerPanic.getStrCode())
+ }
+ }()
+
+ for !w.stop {
+ select {
+ case req, ok := <-w.cmdChan:
+ if !ok {
+ continue
+ }
+ switch r := req.(type) {
+ case *closeReq:
+ w.handleClose(r)
+ }
+ case req, ok := <-w.dataChan:
+ if !ok {
+ continue
+ }
+ w.handleSendData(req)
+ case <-w.batchTimeoutTicker.C:
+ // handle batch timeout
+ w.handleBatchTimeout()
+ case <-w.sendTimeoutTicker.C:
+ // handle send timeout
+ w.handleSendTimeout()
+ case <-w.mapCleanTicker.C:
+ // clean unackedBatches periodically, in golang, map will get bigger and bigger
+ w.handleCleanMap()
+ case <-w.heartbeatTicker.C:
+ // send heartbeat periodically
+ w.handleSendHeartbeat()
+ case <-w.updateConnTicker.C:
+ // update connection periodically
+ w.handleUpdateConn()
+ case batch, ok := <-w.sendFailedBatches:
+ // handle send failed batches
+ if !ok {
+ continue
+ }
+ w.handleSendFailed(batch)
+ case batch, ok := <-w.retryBatches:
+ // handle retry batches
+ if !ok {
+ continue
+ }
+ w.handleRetry(batch, true)
+ case rsp, ok := <-w.responseBatches:
+ // handle responses
+ if !ok {
+ continue
+ }
+ w.handleRsp(rsp)
+ }
+ }
+ }()
+}
+
+func (w *worker) doSendAsync(ctx context.Context, msg Message, callback Callback, flushImmediately bool) {
+ req := &sendDataReq{
+ ctx: ctx,
+ msg: msg,
+ callback: callback,
+ flushImmediately: flushImmediately,
+ publishTime: time.Now(),
+ metrics: w.metrics,
+ workerID: w.indexStr,
+ }
+
+ if len(msg.Payload) == 0 {
+ req.done(errBadLog, "")
+ return
+ }
+
+ // worker is closed
+ if w.getState() != stateReady {
+ req.done(errProducerClosed, "")
+ return
+ }
+
+ // use a semaphore to trace if the data channel is full
+ if w.options.BlockIfQueueIsFull {
+ if !w.dataSemaphore.Acquire(ctx) {
+ w.log.Warn("queue is full, worker index:", w.index)
+ req.done(errContextExpired, "")
+ return
+ }
+ } else {
+ if !w.dataSemaphore.TryAcquire() {
+ w.log.Warn("queue is full, worker index:", w.index)
+ req.done(errSendQueueIsFull, "")
+ return
+ }
+ }
+
+ // save the semaphore, when the request is done, release semaphore
+ req.semaphore = w.dataSemaphore
+ w.dataChan <- req
+ w.metrics.incPending(w.indexStr)
+}
+
+func (w *worker) send(ctx context.Context, msg Message) error {
+ var err error
+
+ // trace if the message is done
+ isDone := atomic.NewBool(false)
+ doneCh := make(chan struct{})
+
+ w.doSendAsync(ctx, msg, func(msg Message, e error) {
+ if isDone.CompareAndSwap(false, true) {
+ err = e // save the error
+ close(doneCh) // notify that the message is done
+ }
+ }, true)
+
+ // wait for send done
+ <-doneCh
+ return err
+}
+
+func (w *worker) sendAsync(ctx context.Context, msg Message, callback Callback) {
+ w.doSendAsync(ctx, msg, callback, false)
+}
+
+func (w *worker) handleSendData(req *sendDataReq) {
+ // w.log.Debug("worker[", w.index, "] handleSendData")
+ // only the messages that with the same stream ID can be sent in a batch, we use the stream ID as the key
+ batch, ok := w.pendingBatches[req.msg.StreamID]
+ if !ok {
+ streamID := req.msg.StreamID
+ batch = &batchReq{
+ workerID: w.indexStr,
+ batchID: util.SnowFlakeID(),
+ groupID: w.options.GroupID,
+ streamID: streamID,
+ dataReqs: make([]*sendDataReq, 0, w.options.BatchingMaxMessages),
+ batchTime: time.Now(),
+ retries: 0,
+ bufferPool: w.bufferPool,
+ bytePool: w.bytePool,
+ metrics: w.metrics,
+ addColumns: w.options.addColumnStr,
+ }
+ w.log.Debug("worker[", w.index, "] new a batch:", batch.batchID, ", streamID:", batch.streamID)
+ w.pendingBatches[streamID] = batch
+ }
+
+ // batch is a pointer, update it directly
+ batch.append(req)
+
+ // no need to flush immediately, not reach the batching max message number/size, just wait
+ if !req.flushImmediately &&
+ len(batch.dataReqs) < w.options.BatchingMaxMessages &&
+ batch.dataSize < w.options.BatchingMaxSize {
+ return
+ }
+
+ // send and delete from the pending batches map
+ w.sendBatch(batch, true)
+ delete(w.pendingBatches, batch.streamID)
+}
+
+func (w *worker) sendBatch(b *batchReq, retryOnFail bool) {
+ // w.log.Debug("worker[", w.index, "] sendBatch")
+ b.lastSendTime = time.Now()
+ b.encode()
+
+ //error callback
+ onErr := func(e error) {
+ defer func() {
+ if rec := recover(); rec != nil {
+ w.log.Error("panic:", rec)
+ w.log.Error(string(debug.Stack()))
+ w.metrics.incError(errServerPanic.getStrCode())
+ }
+ }()
+
+ w.metrics.incError(errConnWriteFailed.getStrCode())
+ w.log.Error("send batch failed, err:", e)
+
+ // close already
+ if w.getState() == stateClosed {
+ b.done(errConnWriteFailed)
+ return
+ }
+
+ // important:when AsyncWrite() call succeed, the batch will be put into w.unackedBatches,now it failed, we need
+ // to delete from w.unackedBatches, as onErr() is call concurrently in different goroutine, we can not delete it
+ // from this callback directly, or will be panic, so we put into the w.sendFailedBatches channel, and it will be
+ // deleted in handleSendFailed() one by one
+ w.sendFailedBatches <- b
+
+ // network error, change a new connection
+ w.updateConn(errConnWriteFailed)
+ // put the batch to the retry channel
+ if retryOnFail {
+ // w.retryBatches <- b
+ w.backoffRetry(context.Background(), b)
+ } else {
+ b.done(errConnWriteFailed)
+ }
+ }
+
+ // w.log.Debug("worker[", w.index, "] write to:", conn.RemoteAddr())
+ // very important:'cause we use gnet, we must call AsyncWrite to send data in goroutines that are different from gnet.OnTraffic() callback
+ conn := w.getConn()
+ err := conn.AsyncWrite(b.buffer.Bytes(), func(c gnet.Conn, e error) error {
+ if e != nil {
+ onErr(e) //error callback
+ }
+ return nil
+ })
+
+ if err != nil {
+ onErr(err) //error callback
+ return
+ }
+
+ // important:as AsyncWrite() is async,AsyncWrite() call succeed doesn't mean send succeed, however, we put the batch
+ // into unackedBatches in advance, if AsyncWrite send failed finally, the batch will be deleted from unackedBatches
+ // in the callback onErr(), if AsyncWrite() send succeed finally, the batch will be deleted from unackedBatches when
+ // the response is received or timeout
+ w.unackedBatchCount++
+ w.unackedBatches[b.batchID] = b
+}
+
+func (w *worker) handleSendFailed(b *batchReq) {
+ // send failed, delete the batch from unackedBatches
+ delete(w.unackedBatches, b.batchID)
+}
+
+func (w *worker) backoffRetry(ctx context.Context, batch *batchReq) {
+ if batch.retries >= w.options.MaxRetries {
+ batch.done(errSendTimeout)
+ w.log.Debug("to many reties, batch done:", batch.batchID)
+ return
+ }
+
+ // it is closed already
+ if w.getState() == stateClosed {
+ batch.done(errSendTimeout)
+ return
+ }
+
+ go func() {
+ defer func() {
+ if rec := recover(); rec != nil {
+ w.log.Error("panic:", rec)
+ w.log.Error(string(debug.Stack()))
+ w.metrics.incError(errServerPanic.getStrCode())
+ }
+ }()
+
+ minBackoff := 100 * time.Millisecond
+ maxBackoff := 10 * time.Second
+ jitterPercent := 0.2
+
+ backoff := time.Duration(batch.retries+1) * minBackoff
+ if backoff > maxBackoff {
+ backoff = maxBackoff
+ }
+
+ // rand will be panic in concurrent call, so we create a new one at each call
+ jitterRand := rand.New(rand.NewSource(time.Now().UnixNano()))
+ jitter := jitterRand.Float64() * float64(backoff) * jitterPercent
+ backoff += time.Duration(jitter)
+
+ select {
+ case <-time.After(backoff):
+ // check if the worker is closed again
+ if w.getState() == stateClosed {
+ batch.done(errSendTimeout)
+ return
+ }
+
+ // put the batch into the retry channel
+ w.retryBatches <- batch
+ case <-ctx.Done():
+ // in the case the process exit, just end up the batch sending routine
+ batch.done(errSendTimeout)
+ }
+ }()
+}
+
+func (w *worker) handleRetry(batch *batchReq, retryOnFail bool) {
+ batch.retries++
+ if batch.retries >= w.options.MaxRetries {
+ batch.done(errSendTimeout)
+ w.log.Debug("to many reties, batch done:", batch.batchID)
+ return
+ }
+
+ // retry
+ w.metrics.incRetry(w.indexStr)
+ w.sendBatch(batch, retryOnFail)
+}
+
+func (w *worker) handleBatchTimeout() {
+ for streamID, batch := range w.pendingBatches {
+ if time.Since(batch.batchTime) > w.options.BatchingMaxPublishDelay {
+ w.log.Debug("worker[", w.index, "] batch timeout, send it now:", batch.batchID, ", streamID:", streamID)
+ w.sendBatch(batch, true)
+ delete(w.pendingBatches, batch.streamID)
+ }
+ }
+}
+
+func (w *worker) handleSendTimeout() {
+ // here may be ineffective
+ for batchID, batch := range w.unackedBatches {
+ if time.Since(batch.lastSendTime) > w.options.SendTimeout {
+ w.log.Warn("worker[", w.index, "] send timeout, resend it now:", batch.batchID, "batchID:", batchID,
+ ",last send time:", batch.lastSendTime.UnixMilli(), ", now:", time.Now().UnixMilli(), "timeout option:", w.options.SendTimeout)
+ //
+ // w.retryBatches <- batch
+ w.backoffRetry(context.Background(), batch)
+ // as retry will put it back to unackedBatches, we delete it here
+ delete(w.unackedBatches, batchID)
+ w.metrics.incTimeout(w.indexStr)
+ }
+ }
+}
+
+func (w *worker) handleCleanMap() {
+ // clean it when we write the map more than 500000 times
+ if w.unackedBatchCount < defaultMapCleanThreshold {
+ return
+ }
+
+ w.log.Debug("clean map")
+ // create a new map and copy the data from the old map
+ newMap := make(map[string]*batchReq)
+ for k, v := range w.unackedBatches {
+ newMap[k] = v
+ }
+
+ // update the map with the new map
+ w.unackedBatches = newMap
+ // reset the counter
+ w.unackedBatchCount = 0
+}
+
+func (w *worker) handleSendHeartbeat() {
+ hb := heartbeatReq{}
+ bb := w.bufferPool.Get()
+ bytes := hb.encode(bb)
+
+ onErr := func(e error) {
+ w.metrics.incError(errConnWriteFailed.getStrCode())
+ w.log.Error("send heartbeat failed, err:", e)
+ w.updateConn(errConnWriteFailed)
+ }
+
+ // very important:'cause we use gnet, we must call AsyncWrite to send data in goroutines that are different from gnet.OnTraffic() callback
+ conn := w.getConn()
+ err := conn.AsyncWrite(bytes, func(c gnet.Conn, e error) error {
+ if e != nil {
+ onErr(e)
+ }
+ // recycle the buffer
+ w.bufferPool.Put(bb)
+ return nil
+ })
+
+ if err != nil {
+ onErr(err)
+ // recycle the buffer
+ w.bufferPool.Put(bb)
+ }
+}
+
+func (w *worker) onRsp(rsp batchRsp) {
+ // close already
+ if w.getState() == stateClosed {
+ return
+ }
+ w.responseBatches <- rsp
+}
+
+func (w *worker) handleRsp(rsp batchRsp) {
+ batchID := rsp.batchID
+ batch, ok := w.unackedBatches[batchID]
+ if !ok {
+ w.log.Warn("worker[", w.index, "] batch not found in unackedBatches map:", batchID, ", send time:", rsp.dt, ", now:", time.Now().UnixMilli())
+ w.metrics.incError(errNoMatchReq4Rsp.strCode)
+ return
+ }
+
+ /*
+ w.log.Debug("worker[", w.index, "] batch done:", batchID, ", batch time:", batch.batchTime.UnixMilli(),
+ ", batch last send time:", batch.lastSendTime.UnixMilli(), ", now:", time.Now().UnixMilli(),
+ "batch retry:", batch.retries)
+ */
+ // call batch.done to release the resources it holds
+ var err = error(nil)
+ if rsp.errCode != 0 {
+ err = errServerError
+ w.log.Error("send succeed but got error code:", rsp.errCode)
+ }
+ batch.done(err)
+ delete(w.unackedBatches, batchID)
+}
+
+func (w *worker) close() {
+ // closed already
+ if w.getState() != stateReady {
+ return
+ }
+
+ req := &closeReq{
+ doneCh: make(chan struct{}),
+ }
+ // new a close request and put it to the command channel
+ w.cmdChan <- req
+
+ // wait for the close request done
+ <-req.doneCh
+ w.stop = true
+}
+
+func (w *worker) handleClose(req *closeReq) {
+ if !w.casState(stateReady, stateClosing) {
+ close(req.doneCh)
+ return
+ }
+
+ // stop the batch timeout ticker, all the pending message will be sent immediately
+ w.batchTimeoutTicker.Stop()
+ // stop the map clean ticker
+ w.mapCleanTicker.Stop()
+ // w.sendTimeoutTicker not stop, sending timeout still need to be handled
+ //
+ // stop the connection updating ticker
+ w.updateConnTicker.Stop()
+
+ // consume the message in w.dataChan channel, first, we close dataChan in a new goroutine
+ // when no message remains, the for loop will break
+ go func() {
+ close(w.dataChan)
+ }()
+ for s := range w.dataChan {
+ w.handleSendData(s)
+ }
+
+ // now, all the messages that are not sent are in w.pendingBatches, consume them, send them immediately, and just send
+ // only one time, without retrying
+ for streamID, batch := range w.pendingBatches {
+ delete(w.pendingBatches, streamID)
+ w.sendBatch(batch, false) // no retry anymore
+ }
+
+ // as we are sending asynchronously, the message that are sent and failed need to update w.retryBatches, we can't
+ // close it now
+ for i := 0; i < len(w.retryBatches); i++ {
+ r := <-w.retryBatches
+ w.handleRetry(r, false) // no retry anymore
+ }
+
+ // now, only w.unackedBatches still has batches, these batches haven't received a response, assign a callback to them,
+ // when all these batches receive a response or timeout, close all the resources. 'case there is no other goroutine is
+ // updating w.unackedBatches, it safe to update it here.
+
+ // get the left unacknowledged batches
+ left := atomic.NewInt32(int32(len(w.unackedBatches)))
+ w.log.Debug("worker:", w.index, "unacked:", left.Load())
+
+ closeAll := func() {
+ // stop the send timeout ticker
+ w.sendTimeoutTicker.Stop()
+ // release the connection
+ w.client.putConn(w.getConn(), nil)
+ // close the command channel
+ close(w.cmdChan)
+ // update the worker state to stateClosed, from now on, w.retryBatches/w.sendFailedBatches/w.responseBatches can't
+ // be written anymore, or it will be panic, so, when before updating these channels, we need to check if the state
+ // is stateClosed, if it is, stop updating
+ w.setState(stateClosed)
+ // close the retry channel
+ close(w.retryBatches)
+ // close the send failed channel
+ close(w.sendFailedBatches)
+ // close the response channel
+ close(w.responseBatches)
+ // close the done channel of the close request to notify the close is done
+ close(req.doneCh)
+ }
+
+ // no left batches, just close and return
+ if left.Load() <= 0 {
+ w.log.Debug("no batch left, close now")
+ closeAll()
+ return
+ }
+
+ for id, batch := range w.unackedBatches {
+ // update the left batches, add a callback to it
+ batch.callback = func() {
+ // when receive a response or timeout, decrease the counter by 1, when left <=0, indicates that all are done
+ l := left.Add(-1)
+ if l <= 0 {
+ w.log.Debug("left batches all done, close now")
+ closeAll()
+ }
+ }
+ // store it back to the map
+ w.unackedBatches[id] = batch
+ }
+}
+
+func (w *worker) handleUpdateConn() {
+ w.updateConn(nil)
+}
+
+func (w *worker) updateConn(err error) {
+ w.log.Debug("worker[", w.index, "] updateConn")
+ newConn, newErr := w.client.getConn()
+ if newErr != nil {
+ w.log.Error("get new conn error:", newErr)
+ w.metrics.incError(errNewConnFailed.getStrCode())
+ return
+ }
+
+ oldConn := w.getConn()
+ w.client.putConn(oldConn, err)
+ w.setConn(newConn)
+ w.metrics.incUpdateConn(getErrorCode(err))
+}
+
+func (w *worker) setConn(conn gnet.Conn) {
+ w.conn.Store(conn)
+}
+
+func (w *worker) getConn() gnet.Conn {
+ return w.conn.Load().(gnet.Conn)
+}
+
+func (w *worker) setState(state workerState) {
+ w.state.Swap(int32(state))
+}
+
+func (w *worker) getState() workerState {
+ return workerState(w.state.Load())
+}
+
+func (w *worker) casState(oldState, newState workerState) bool {
+ return w.state.CompareAndSwap(int32(oldState), int32(newState))
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/worker_test.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/worker_test.go
new file mode 100755
index 000000000..e2ec291f5
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy/worker_test.go
@@ -0,0 +1,77 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dataproxy
+
+import (
+ "errors"
+ "testing"
+)
+
+func Test_getErrorCode(t *testing.T) {
+ type args struct {
+ err error
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ }{
+ // TODO: Add test cases.
+ {
+ name: "nil",
+ args: args{err: nil},
+ want: errOK.strCode,
+ },
+ {
+ name: "errSendTimeout",
+ args: args{err: errSendTimeout},
+ want: errSendTimeout.strCode,
+ },
+ {
+ name: "errSendFailed",
+ args: args{err: errSendFailed},
+ want: errSendFailed.strCode,
+ },
+ {
+ name: "errProducerClosed",
+ args: args{err: errProducerClosed},
+ want: errProducerClosed.strCode,
+ },
+ {
+ name: "errSendQueueIsFull",
+ args: args{err: errSendQueueIsFull},
+ want: errSendQueueIsFull.strCode,
+ },
+ {
+ name: "errContextExpired",
+ args: args{err: errContextExpired},
+ want: errContextExpired.strCode,
+ },
+ {
+ name: "unknown",
+ args: args{err: errors.New("unknown")},
+ want: errUnknown.strCode,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := getErrorCode(tt.args.err); got != tt.want {
+ t.Errorf("getErrorCode() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/discoverer.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/discoverer.go
new file mode 100755
index 000000000..96d79ce93
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/discoverer.go
@@ -0,0 +1,88 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discoverer
+
+import (
+ "net"
+ "strconv"
+)
+
+// Endpoint is the config of an endpoint
+type Endpoint struct {
+ Addr string
+ Host string
+ Port int
+}
+
+// EndpointList is the list of endpoints
+type EndpointList []Endpoint
+
+// Addresses returns the addresses of an endpoint list
+func (el EndpointList) Addresses() []string {
+ addrs := make([]string, 0, len(el))
+ for _, endpoint := range el {
+ addrs = append(addrs, endpoint.Addr)
+ }
+
+ return addrs
+}
+
+// AddressMap returns the addresses map of an endpoint list
+func (el EndpointList) AddressMap() map[string]struct{} {
+ addrs := make(map[string]struct{})
+ for _, endpoint := range el {
+ addrs[endpoint.Addr] = struct{}{}
+ }
+
+ return addrs
+}
+
+// EventHandler is the interface of the discoverer
+type EventHandler interface {
+ OnEndpointUpdate(all, add, del EndpointList)
+}
+
+// EndpointProvider is the interface of an endpoint provider
+type EndpointProvider interface {
+ // GetEndpoints gets all endpoints from the load balancer, since the discoverer will update internally,
+ // the caller should copy the endpoint list to its own routine space
+ GetEndpoints() EndpointList
+}
+
+// Discoverer is the interface of a service discoverer
+type Discoverer interface {
+ EndpointProvider
+
+ // AddEventHandler adds an EventHandler to the discoverer
+ AddEventHandler(h EventHandler)
+
+ // DelEventHandler remove an EventHandler from the discoverer
+ DelEventHandler(h EventHandler)
+
+ // Close closes the discoverer
+ Close()
+}
+
+// BuildAddr builds a host address
+func BuildAddr(host string, port int) string {
+ return net.JoinHostPort(host, strconv.Itoa(port))
+}
+
+// BuildAddrString builds a host address
+func BuildAddrString(host, port string) string {
+ return net.JoinHostPort(host, port)
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/dns.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/dns.go
new file mode 100755
index 000000000..1e8d321aa
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/dns.go
@@ -0,0 +1,223 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discoverer
+
+import (
+ "errors"
+ "net"
+ "regexp"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger"
+)
+
+const (
+ // IPRegexp is the regular expression for IP address
+ IPRegexp = `^((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-fA-F]|[a-fA-F][a-fA-F0-9\-]*[a-fA-F0-9])\.)*([A-Fa-f]|[A-Fa-f][A-Fa-f0-9\-]*[A-Fa-f0-9])$|^(?:(?:(?:(?:(?:(?:(?:[0-9a-fA-F]{1,4})):){6})(?:(?:(?:(?:(?:[0-9a-fA-F]{1,4})):(?:(?:[0-9a-fA-F]{1,4})))|(?:(?:(?:(?:(?:25[0-5]|(?:[1-9]|1[0-9]|2[0-4])?[0-9]))\.){3}(?:(?:25[0-5]|(?:[1-9]|1[0-9]|2[0-4])?[0-9])))))))|(?:(?:::(?:(?:(?:[0-9a-fA-F]{1,4})):){5})(?:(?:(?:(?:(?:[0- [...]
+)
+
+// variables
+var (
+ ErrInvalidPort = errors.New("invalid port")
+ ErrNoAvailableEndpoint = errors.New("no available endpoints")
+)
+
+// NewDNS create a DNS discoverer
+func NewDNS(domain string, port int, lookupInterval time.Duration, log logger.Logger) (Discoverer, error) {
+ logger := logger.Std()
+ if log != nil {
+ logger = log
+ }
+
+ if port <= 0 || port > 65535 {
+ return nil, ErrInvalidPort
+ }
+
+ if lookupInterval == 0 {
+ lookupInterval = 30 * time.Second
+ }
+
+ dis := &dnsDiscoverer{
+ domain: domain,
+ port: port,
+ lookupInterval: lookupInterval,
+ endpointList: make([]Endpoint, 0),
+ hostListMap: make(map[string]struct{}),
+ eventHandlers: make(map[EventHandler]struct{}),
+ log: logger,
+ }
+
+ regex := regexp.MustCompile(IPRegexp)
+ // input domain is an IP
+ if regex.MatchString(domain) {
+ ip := net.ParseIP(domain)
+ dis.endpointList = []Endpoint{{Host: ip.String(), Port: port, Addr: BuildAddr(ip.String(), port)}}
+ dis.hostListStr = ip.String()
+
+ return dis, nil
+ }
+
+ // domain
+ dis.lookup()
+ dis.update()
+ return dis, nil
+}
+
+type dnsDiscoverer struct {
+ sync.RWMutex
+ domain string
+ port int
+ lookupInterval time.Duration
+ endpointList EndpointList
+ hostListStr string
+ hostListMap map[string]struct{}
+ eventHandlers map[EventHandler]struct{}
+ closeFunc func()
+ log logger.Logger
+}
+
+func (d *dnsDiscoverer) GetEndpoints() EndpointList {
+ d.RLock()
+ defer d.RUnlock()
+
+ return d.endpointList
+}
+
+func (d *dnsDiscoverer) AddEventHandler(h EventHandler) {
+ d.Lock()
+ defer d.Unlock()
+
+ d.eventHandlers[h] = struct{}{}
+}
+
+func (d *dnsDiscoverer) DelEventHandler(h EventHandler) {
+ d.Lock()
+ defer d.Unlock()
+
+ delete(d.eventHandlers, h)
+}
+
+func (d *dnsDiscoverer) Close() {
+ if d.closeFunc != nil {
+ d.closeFunc()
+ }
+}
+
+func (d *dnsDiscoverer) lookup() {
+ hosts := make(map[string]struct{}, 16)
+ for i := 1; i <= 32; i++ {
+ lookupHosts, err := net.LookupHost(d.domain)
+ if err != nil {
+ d.log.Errorf("domain lookup failed: %v", err)
+ break
+ }
+
+ for _, host := range lookupHosts {
+ hosts[host] = struct{}{}
+ }
+
+ if len(lookupHosts) > 1 {
+ // discoverer server return a list of hosts
+ break
+ } else {
+ // discoverer server return only one ip per request
+ if i > len(hosts)*2 && i > 12 {
+ break
+ }
+ }
+ }
+ // will not update if ip list is empty
+ if len(hosts) == 0 {
+ d.log.Warnf("no hosts were found from domain: %s, we will keep the local cache", d.domain)
+ return
+ }
+
+ allHosts := make([]string, 0, len(hosts))
+ allHostMap := make(map[string]struct{})
+ allEndpoints := make([]Endpoint, 0, len(hosts))
+ addEndpoints := make([]Endpoint, 0)
+ for host := range hosts {
+ allHosts = append(allHosts, host)
+ allHostMap[host] = struct{}{}
+ allEndpoints = append(allEndpoints, Endpoint{Host: host, Port: d.port, Addr: BuildAddr(host, d.port)})
+
+ // new host is not found in the old host list map, it is a new added one
+ if _, ok := d.hostListMap[host]; !ok {
+ addEndpoints = append(addEndpoints, Endpoint{Host: host, Port: d.port, Addr: BuildAddr(host, d.port)})
+ }
+ }
+
+ delEndpoints := make([]Endpoint, 0)
+ for host := range d.hostListMap {
+ // old host is not found in the new host list map, it is deleted
+ if _, ok := allHostMap[host]; !ok {
+ delEndpoints = append(delEndpoints, Endpoint{Host: host, Port: d.port, Addr: BuildAddr(host, d.port)})
+ }
+ }
+
+ d.Lock()
+ d.endpointList = allEndpoints
+ d.hostListMap = allHostMap
+ d.Unlock()
+
+ // show this logger only in case of ip list change
+ // only ticker will update d.allHostStr, lock is unnecessary
+ sort.Strings(allHosts)
+ allHostStr := strings.Join(allHosts, ";")
+ if allHostStr != d.hostListStr {
+ d.hostListStr = allHostStr
+ d.log.Infof("update domain host list %s: %v", d.domain, allHostStr)
+ }
+
+ d.RLock()
+ defer d.RUnlock()
+ if len(addEndpoints) > 0 || len(delEndpoints) > 0 {
+ for h := range d.eventHandlers {
+ h.OnEndpointUpdate(allEndpoints, addEndpoints, delEndpoints)
+ }
+ }
+}
+
+func (d *dnsDiscoverer) update() {
+ wg := sync.WaitGroup{}
+ ticker := time.NewTicker(d.lookupInterval)
+ stopCh := make(chan struct{})
+
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case <-ticker.C:
+ d.lookup()
+ case <-stopCh:
+ return
+ default:
+ time.Sleep(500 * time.Millisecond)
+ }
+ }
+ }()
+
+ d.closeFunc = func() {
+ ticker.Stop()
+ close(stopCh)
+ wg.Wait()
+ }
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/dns_test.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/dns_test.go
new file mode 100755
index 000000000..d2f36b509
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/discoverer/dns_test.go
@@ -0,0 +1,42 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package discoverer
+
+import (
+ "fmt"
+ "testing"
+ "time"
+)
+
+type handler struct {
+}
+
+func (h *handler) OnEndpointUpdate(all, add, del EndpointList) {
+ fmt.Println("all:", all)
+ fmt.Println("add:", add)
+ fmt.Println("del:", del)
+}
+
+func TestNewDNS(t *testing.T) {
+ dns, err := NewDNS("dev.tglog.com", 20001, 5*time.Second, nil)
+ if err != nil {
+ t.Error(err)
+ }
+ fmt.Println("get", dns.GetEndpoints())
+ dns.AddEventHandler(&handler{})
+ select {}
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/framer/framer.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/framer/framer.go
new file mode 100755
index 000000000..ee54c4fce
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/framer/framer.go
@@ -0,0 +1,171 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package framer
+
+import (
+ "encoding/binary"
+ "errors"
+)
+
+// Framer types and other constants
+const (
+ defaultSize = 64 * 1024
+)
+
+// Framer errors
+var (
+ ErrIncompleteFrame = errors.New("incomplete frame")
+ ErrInvalidFrameLen = errors.New("invalid frame length")
+ ErrInvalidFrameLenCfg = errors.New("invalid field length for length field base Framer, expect(1, 2, 3, 4, 8)")
+ ErrExceedMaxFrameLen = errors.New("exceed max frame length")
+ ErrFrameLenLessThanLenFieldEndOffset = errors.New("frame length less then length field end offset")
+ ErrNoEnoughBytesToTrip = errors.New("no enough bytes to trip")
+)
+
+// Framer is the interface of a stream Framer
+type Framer interface {
+ // ReadFrame reads a frame from the raw stream input according to its config,
+ // return the frame length, the payload beginning and ending offset
+ ReadFrame(input []byte) (frameLen, payloadOffset, payloadOffsetEnd int, err error)
+}
+
+// LengthFieldCfg is the config of a length field base framer
+type LengthFieldCfg struct {
+ LittleEndian bool // use little endian or not, default: false
+ MaxFrameLen int // max frame length in bytes
+ FieldOffset int // the offset of the length field
+ FieldLength int // the length of the length field itself
+ Adjustment int // adjustment value, can be positive or negative
+ BytesToStrip int // bytes to strip
+}
+
+type lengthFieldBased struct {
+ byteOrder binary.ByteOrder
+ cfg LengthFieldCfg
+}
+
+// NewLengthField news a length field based framer
+func NewLengthField(cfg LengthFieldCfg) (Framer, error) {
+ if cfg.FieldOffset < 0 {
+ return nil, errors.New("invalid field offset for length field base Framer")
+ }
+
+ if cfg.FieldLength <= 0 {
+ return nil, errors.New("invalid field length for length field base Framer")
+ }
+
+ if cfg.BytesToStrip < 0 {
+ return nil, errors.New("invalid bytes to trip for length field base Framer")
+ }
+
+ if cfg.FieldLength != 1 && cfg.FieldLength != 2 && cfg.FieldLength != 3 && cfg.FieldLength != 4 && cfg.FieldLength != 8 {
+ return nil, ErrInvalidFrameLenCfg
+ }
+
+ if cfg.MaxFrameLen <= 0 {
+ cfg.MaxFrameLen = defaultSize
+ }
+
+ if cfg.FieldOffset+cfg.FieldLength >= cfg.MaxFrameLen {
+ return nil, errors.New("invalid field offset and max frame length for length field base Framer")
+ }
+
+ var byteOrder binary.ByteOrder = binary.BigEndian
+ if cfg.LittleEndian {
+ byteOrder = binary.LittleEndian
+ }
+
+ return &lengthFieldBased{byteOrder: byteOrder, cfg: cfg}, nil
+}
+
+func (c *lengthFieldBased) ReadFrame(input []byte) (frameLen, payloadOffset, payloadOffsetEnd int, err error) {
+ inLen := len(input)
+ if inLen <= 0 {
+ return 0, 0, 0, ErrIncompleteFrame
+ }
+
+ // length field end offset
+ fieldEndOffset := c.cfg.FieldOffset + c.cfg.FieldLength
+
+ // input length is not enough to read the length of a frame
+ if inLen < fieldEndOffset {
+ return 0, 0, 0, ErrIncompleteFrame
+ }
+
+ // length field buffer
+ lenFieldBuf := input[c.cfg.FieldOffset:fieldEndOffset]
+ // get the unadjusted length
+ var unadjustedLen uint64
+ switch c.cfg.FieldLength {
+ case 1:
+ unadjustedLen = uint64(lenFieldBuf[0])
+ case 2:
+ unadjustedLen = uint64(c.byteOrder.Uint16(lenFieldBuf))
+ case 3:
+ unadjustedLen = uint24(c.byteOrder, lenFieldBuf)
+ case 4:
+ unadjustedLen = uint64(c.byteOrder.Uint32(lenFieldBuf))
+ case 8:
+ unadjustedLen = c.byteOrder.Uint64(lenFieldBuf)
+ default:
+ return 0, 0, 0, ErrInvalidFrameLenCfg
+ }
+
+ // the unadjusted length is invalid
+ if unadjustedLen <= 0 {
+ return 0, 0, 0, ErrInvalidFrameLen
+ }
+
+ // get the adjusted length
+ frameLen = fieldEndOffset + c.cfg.Adjustment + int(unadjustedLen)
+ // adjusted length less than the length field end offset, no payload
+ if frameLen < fieldEndOffset {
+ return 0, 0, 0, ErrFrameLenLessThanLenFieldEndOffset
+ }
+
+ // adjusted length greater than the max frame length
+ if frameLen > c.cfg.MaxFrameLen {
+ return 0, 0, 0, ErrExceedMaxFrameLen
+ }
+
+ // adjusted length greater than the input length, input is not enough
+ if frameLen > inLen {
+ return 0, 0, 0, ErrIncompleteFrame
+ }
+
+ // no enough bytes to trip
+ if c.cfg.BytesToStrip > frameLen {
+ return 0, 0, 0, ErrNoEnoughBytesToTrip
+ }
+
+ // the actual frame length we need
+ actualFrameLen := frameLen - c.cfg.BytesToStrip
+ // the actual frame offset
+ actualFrameOffset := c.cfg.BytesToStrip
+ // actual frame end offset
+ actualFrameOffsetEnd := actualFrameOffset + actualFrameLen
+
+ return frameLen, actualFrameOffset, actualFrameOffsetEnd, nil
+}
+
+func uint24(byteOrder binary.ByteOrder, b []byte) uint64 {
+ _ = b[2]
+ if byteOrder == binary.LittleEndian {
+ return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16
+ }
+ return uint64(b[2]) | uint64(b[1])<<8 | uint64(b[0])<<16
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/go.mod b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/go.mod
new file mode 100755
index 000000000..e24944731
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/go.mod
@@ -0,0 +1,51 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+module github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang
+
+go 1.17
+
+require (
+ github.com/bwmarrin/snowflake v0.3.0
+ github.com/go-resty/resty/v2 v2.7.0
+ github.com/gofrs/uuid v4.4.0+incompatible
+ github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c
+ github.com/panjf2000/gnet/v2 v2.2.9
+ github.com/prometheus/client_golang v1.14.0
+ github.com/zentures/cityhash v0.0.0-20131128155616-cdd6a94144ab
+)
+
+require (
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/cespare/xxhash/v2 v2.1.2 // indirect
+ github.com/golang/protobuf v1.5.2 // indirect
+ github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+ github.com/prometheus/client_model v0.3.0 // indirect
+ github.com/prometheus/common v0.37.0 // indirect
+ github.com/prometheus/procfs v0.8.0 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
+ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
+ golang.org/x/sys v0.8.0 // indirect
+ google.golang.org/protobuf v1.28.1 // indirect
+ gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+)
+
+require (
+ github.com/BurntSushi/toml v1.2.0 // indirect
+ github.com/klauspost/compress v1.15.15
+ go.uber.org/atomic v1.11.0
+ go.uber.org/zap v1.24.0 // indirect
+)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/go.sum b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/go.sum
new file mode 100755
index 000000000..0f1c859fc
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/go.sum
@@ -0,0 +1,556 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
+github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
+github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
+github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
+github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
+github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw=
+github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
+github.com/panjf2000/ants/v2 v2.7.1 h1:qBy5lfSdbxvrR0yUnZfaEDjf0FlCw4ufsbcsxmE7r+M=
+github.com/panjf2000/ants/v2 v2.7.1/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
+github.com/panjf2000/gnet/v2 v2.2.5 h1:yRmp6ye51eDyHYncdhBiO5RuFAGvPvsOl9RTyZJgkHw=
+github.com/panjf2000/gnet/v2 v2.2.5/go.mod h1:Q34YBnJNDFLsVBC4TiGD3uN+imoXrunFnecs/4FYcx4=
+github.com/panjf2000/gnet/v2 v2.2.8 h1:5pt46bsJ6nfnfi2GWvevi7DsNTOwma3mUzBwm0nK8Io=
+github.com/panjf2000/gnet/v2 v2.2.8/go.mod h1:Q34YBnJNDFLsVBC4TiGD3uN+imoXrunFnecs/4FYcx4=
+github.com/panjf2000/gnet/v2 v2.2.9 h1:rmIkaXYtMb2dkgaedojb1uEM2NgVM0jdrnmSNq7F/Vk=
+github.com/panjf2000/gnet/v2 v2.2.9/go.mod h1:Q34YBnJNDFLsVBC4TiGD3uN+imoXrunFnecs/4FYcx4=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
+github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/zentures/cityhash v0.0.0-20131128155616-cdd6a94144ab h1:BD4YbH4Y0ysgbrP9jGuDB0BxkqyTRk6Y70o3D5Z5ayc=
+github.com/zentures/cityhash v0.0.0-20131128155616-cdd6a94144ab/go.mod h1:SvJE1nX57VqPOyqkQGEGcJPWZqeB3FCZ8s7a0uSlG+A=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
+go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
+go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
+go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
+go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
+go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger/logger.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger/logger.go
new file mode 100755
index 000000000..921ee9f22
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/logger/logger.go
@@ -0,0 +1,95 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logger
+
+import "fmt"
+
+// Logger is the logger interface used in this module
+type Logger interface {
+ Debug(args ...interface{})
+ Info(args ...interface{})
+ Warn(args ...interface{})
+ Error(args ...interface{})
+ Fatal(args ...interface{})
+ Panic(args ...interface{})
+ Debugf(template string, args ...interface{})
+ Infof(template string, args ...interface{})
+ Warnf(template string, args ...interface{})
+ Errorf(template string, args ...interface{})
+ Fatalf(template string, args ...interface{})
+ Panicf(template string, args ...interface{})
+}
+
+var (
+ std = stdLogger{}
+)
+
+type stdLogger struct {
+}
+
+func (s stdLogger) Debug(args ...interface{}) {
+ fmt.Println(args...)
+}
+
+func (s stdLogger) Info(args ...interface{}) {
+ fmt.Println(args...)
+}
+
+func (s stdLogger) Warn(args ...interface{}) {
+ fmt.Println(args...)
+}
+
+func (s stdLogger) Error(args ...interface{}) {
+ fmt.Println(args...)
+}
+
+func (s stdLogger) Fatal(args ...interface{}) {
+ fmt.Println(args...)
+}
+
+func (s stdLogger) Panic(args ...interface{}) {
+ fmt.Println(args...)
+}
+
+func (s stdLogger) Debugf(template string, args ...interface{}) {
+ fmt.Println(fmt.Sprintf(template, args...))
+}
+
+func (s stdLogger) Infof(template string, args ...interface{}) {
+ fmt.Println(fmt.Sprintf(template, args...))
+}
+
+func (s stdLogger) Warnf(template string, args ...interface{}) {
+ fmt.Println(fmt.Sprintf(template, args...))
+}
+
+func (s stdLogger) Errorf(template string, args ...interface{}) {
+ fmt.Println(fmt.Sprintf(template, args...))
+}
+
+func (s stdLogger) Fatalf(template string, args ...interface{}) {
+ fmt.Println(fmt.Sprintf(template, args...))
+}
+
+func (s stdLogger) Panicf(template string, args ...interface{}) {
+ fmt.Println(fmt.Sprintf(template, args...))
+}
+
+// Std returns a standard logger that writes logger to the stdOut
+func Std() Logger {
+ return std
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/syncx/semaphore.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/syncx/semaphore.go
new file mode 100755
index 000000000..92a8d3231
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/syncx/semaphore.go
@@ -0,0 +1,124 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// note: this file is copied and modified from https://github.com/apache/pulsar-client-go/blob/master/pulsar/internal/semaphore.go
+//
+
+package syncx
+
+import (
+ "context"
+ "sync/atomic"
+)
+
+// Semaphore is the interface of a semaphore, this semaphore can acquire one permit each time
+type Semaphore interface {
+ // Acquire a permit, if one is available and returns immediately,
+ // reducing the number of available permits by one.
+ Acquire(ctx context.Context) bool
+
+ // TryAcquire Try to acquire a permit. The method will return immediately
+ // with a `true` if it was possible to acquire a permit and
+ // `false` otherwise.
+ TryAcquire() bool
+
+ // Release a permit, returning it to the semaphore.
+ // Release a permit, increasing the number of available permits by
+ // one. If any threads are trying to acquire a permit, then one is
+ // selected and given the permit that was just released. That thread
+ // is (re)enabled for thread scheduling purposes.
+ // There is no requirement that a thread that releases a permit must
+ // have acquired that permit by calling Acquire().
+ // Correct usage of a semaphore is established by programming convention
+ // in the application.
+ Release()
+
+ // Cap gets the max permits of a semaphore
+ Cap() int32
+
+ // Available gets the available permits of a semaphore, it just a snapshot, not an
+ // accurate value.
+ Available() int32
+}
+
+type semaphore struct {
+ maxPermits int32
+ permits int32
+ ch chan bool
+}
+
+// NewSemaphore news a Semaphore
+func NewSemaphore(maxPermits int32) Semaphore {
+ if maxPermits <= 0 {
+ panic("max permits for semaphore needs to be > 0")
+ }
+
+ return &semaphore{
+ maxPermits: maxPermits,
+ permits: 0,
+ ch: make(chan bool),
+ }
+}
+
+func (s *semaphore) Acquire(ctx context.Context) bool {
+ permits := atomic.AddInt32(&s.permits, 1)
+ if permits <= s.maxPermits {
+ return true
+ }
+
+ // Block on the channel until a new permit is available
+ // or the context expires
+ select {
+ case <-s.ch:
+ return true
+ case <-ctx.Done():
+ atomic.AddInt32(&s.permits, -1)
+ return false
+ }
+}
+
+func (s *semaphore) TryAcquire() bool {
+ for {
+ currentPermits := atomic.LoadInt32(&s.permits)
+ if currentPermits >= s.maxPermits {
+ // All the permits are already exhausted
+ return false
+ }
+
+ if atomic.CompareAndSwapInt32(&s.permits, currentPermits, currentPermits+1) {
+ // Successfully incremented counter
+ return true
+ }
+ }
+}
+
+func (s *semaphore) Release() {
+ permits := atomic.AddInt32(&s.permits, -1)
+ if permits >= s.maxPermits {
+ // Unblock the next in line to acquire the semaphore
+ s.ch <- true
+ }
+}
+
+func (s *semaphore) Cap() int32 {
+ return s.maxPermits
+}
+
+func (s *semaphore) Available() int32 {
+ currentPermits := atomic.LoadInt32(&s.permits)
+ return s.maxPermits - currentPermits
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/bytestring.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/bytestring.go
new file mode 100755
index 000000000..57ff8d714
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/bytestring.go
@@ -0,0 +1,31 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import "unsafe"
+
+// BytesToString without coping
+func BytesToString(bytes []byte) string {
+ return *(*string)(unsafe.Pointer(&bytes))
+}
+
+// StringToBytes without coping
+func StringToBytes(str string) []byte {
+ x := (*[2]uintptr)(unsafe.Pointer(&str))
+ h := [3]uintptr{x[0], x[1], x[1]}
+ return *(*[]byte)(unsafe.Pointer(&h))
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/hash.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/hash.go
new file mode 100755
index 000000000..e48a3ad7c
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/hash.go
@@ -0,0 +1,27 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+// JavaStringHash and Java String.hashCode() equivalent
+func JavaStringHash(s string) uint32 {
+ var h uint32
+ for i, size := 0, len(s); i < size; i++ {
+ h = 31*h + uint32(s[i])
+ }
+
+ return h
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/id.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/id.go
new file mode 100755
index 000000000..e4afc5834
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/id.go
@@ -0,0 +1,59 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+ "log"
+
+ "github.com/bwmarrin/snowflake"
+ "github.com/gofrs/uuid"
+ "github.com/zentures/cityhash"
+)
+
+var (
+ snowflakeNode *snowflake.Node
+)
+
+func init() {
+ ip, err := GetFirstPrivateIP()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ id := IPtoUInt(ip)
+ snowflakeNode, err = snowflake.NewNode(int64(id % 1024))
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+// UInt64UUID generates an uint64 UUID
+func UInt64UUID() (uint64, error) {
+ guid, err := uuid.NewV4()
+ if err != nil {
+ return 0, err
+ }
+
+ bytes := guid.Bytes()
+ length := len(bytes)
+ return cityhash.CityHash64WithSeeds(bytes, uint32(length), 13329145742295551469, 7926974186468552394), nil
+}
+
+// SnowFlakeID generates a snowflake ID
+func SnowFlakeID() string {
+ return snowflakeNode.Generate().String()
+}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/ip.go b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/ip.go
new file mode 100755
index 000000000..708464e59
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/util/ip.go
@@ -0,0 +1,109 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+ "encoding/binary"
+ "fmt"
+ "net"
+ "strconv"
+ "strings"
+)
+
+// IPtoUInt converts an IP to an uint64 integer
+func IPtoUInt(ip string) uint32 {
+ ips := net.ParseIP(ip)
+
+ if len(ips) == 16 {
+ return binary.BigEndian.Uint32(ips[12:16])
+ } else if len(ips) == 4 {
+ return binary.BigEndian.Uint32(ips)
+ }
+ return 0
+}
+
+// IsPrivateIP checks if an IP is a private one, according to some reserved inner IP rules, may be not suitable for all
+func IsPrivateIP(ip string) bool {
+ ipByte := strings.Split(ip, ".")
+ if len(ipByte) != 4 {
+ return false
+ }
+
+ first, err := strconv.Atoi(ipByte[0])
+ if err != nil {
+ return false
+ }
+
+ second, err := strconv.Atoi(ipByte[1])
+ if err != nil {
+ return false
+ }
+
+ third, err := strconv.Atoi(ipByte[2])
+ if err != nil {
+ return false
+ }
+
+ fourth, err := strconv.Atoi(ipByte[3])
+ if err != nil {
+ return false
+ }
+
+ if first == 11 || first == 10 || first == 9 || first == 30 ||
+ (first == 100 && second >= 64 && second <= 127) ||
+ (first == 172 && second >= 16 && second <= 31) ||
+ (first == 192 && second == 168) ||
+ (first == 172 && second == 32 && (third == 0 || third == 1) && fourth >= 1 && fourth <= 128) {
+ return true
+ }
+
+ return false
+}
+
+// GetPrivateIPList gets all the private IPs of the current host
+func GetPrivateIPList() ([]string, error) {
+ ips, err := net.InterfaceAddrs()
+ if err != nil {
+ return nil, err
+ }
+
+ var privateIPs []string
+ for _, ip := range ips {
+ parts := strings.Split(ip.String(), "/")
+ if len(parts) != 2 {
+ return nil, fmt.Errorf("ip %v address format error", ip.String())
+ }
+ if IsPrivateIP(parts[0]) {
+ privateIPs = append(privateIPs, parts[0])
+ }
+ }
+
+ return privateIPs, nil
+}
+
+// GetFirstPrivateIP gets the first private IP of the current host
+func GetFirstPrivateIP() (string, error) {
+ ips, err := GetPrivateIPList()
+ if err != nil {
+ return "", err
+ }
+ if len(ips) == 0 {
+ return "", fmt.Errorf("no private ip")
+ }
+
+ return ips[0], nil
+}
diff --git a/pom.xml b/pom.xml
index 50c0dce0a..9c90fdc12 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1447,6 +1447,7 @@
<exclude>**/tubemq-client-go/go.mod</exclude>
<!-- Go sum -->
<exclude>**/tubemq-client-go/go.sum</exclude>
+ <exclude>**/dataproxy-sdk-golang/go.sum</exclude>
<!-- Docker build-->
<exclude>**/docker/docker-compose/**/**</exclude>