You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/09/12 12:03:44 UTC
[inlong] branch master updated: [INLONG-8896][SDK] Remove old code before refactoring for dataproxy cpp sdk (#8902)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 0a531e99b9 [INLONG-8896][SDK] Remove old code before refactoring for dataproxy cpp sdk (#8902)
0a531e99b9 is described below
commit 0a531e99b9570330e2c28c7f6f2a44a9ffe3885d
Author: doleyzi <43...@users.noreply.github.com>
AuthorDate: Tue Sep 12 20:03:38 2023 +0800
[INLONG-8896][SDK] Remove old code before refactoring for dataproxy cpp sdk (#8902)
---
.../dataproxy-sdk-cpp/CMakeLists.txt | 27 +-
.../release/conf/config_example.json | 2 +-
.../dataproxy-sdk-cpp/release/demo/send_demo.cc | 14 +-
.../dataproxy-sdk-cpp/release/inc/client_config.h | 115 ---
.../dataproxy-sdk-cpp/release/inc/sdk_conf.h | 8 +-
.../dataproxy-sdk-cpp/release/inc/tc_api.h | 118 ---
.../dataproxy-sdk-cpp/release/inc/user_msg.h | 64 --
.../dataproxy-sdk-cpp/src/base/CMakeLists.txt | 26 -
.../dataproxy-sdk-cpp/src/base/atomic.h | 101 ---
.../dataproxy-sdk-cpp/src/base/client_config.cc | 705 ---------------
.../dataproxy-sdk-cpp/src/base/ini_help.cc | 292 ------
.../dataproxy-sdk-cpp/src/base/ini_help.h | 94 --
.../dataproxy-sdk-cpp/src/base/logger.cc | 335 -------
.../dataproxy-sdk-cpp/src/base/logger.h | 244 -----
.../dataproxy-sdk-cpp/src/base/msg_protocol.h | 90 --
.../dataproxy-sdk-cpp/src/base/noncopyable.h | 37 -
.../dataproxy-sdk-cpp/src/base/pack_queue.cc | 580 ------------
.../dataproxy-sdk-cpp/src/base/pack_queue.h | 133 ---
.../dataproxy-sdk-cpp/src/base/proxylist_config.cc | 989 ---------------------
.../dataproxy-sdk-cpp/src/base/proxylist_config.h | 200 -----
.../dataproxy-sdk-cpp/src/base/read_write_mutex.h | 133 ---
.../dataproxy-sdk-cpp/src/base/recv_buffer.h | 419 ---------
.../dataproxy-sdk-cpp/src/base/sdk_constant.h | 105 ---
.../dataproxy-sdk-cpp/src/base/sdk_core.cc | 332 -------
.../dataproxy-sdk-cpp/src/base/sdk_core.h | 64 --
.../dataproxy-sdk-cpp/src/base/singleton.h | 63 --
.../dataproxy-sdk-cpp/src/base/user_msg.h | 63 --
.../dataproxy-sdk-cpp/src/base/utils.cc | 440 ---------
.../dataproxy-sdk-cpp/src/base/utils.h | 85 --
.../dataproxy-sdk-cpp/src/config/sdk_conf.cc | 82 +-
.../dataproxy-sdk-cpp/src/core/api_imp.cc | 9 +-
.../dataproxy-sdk-cpp/src/manager/proxy_manager.cc | 133 +--
.../dataproxy-sdk-cpp/src/manager/proxy_manager.h | 2 +-
.../dataproxy-sdk-cpp/src/manager/send_manager.cc | 8 +-
.../dataproxy-sdk-cpp/src/net/CMakeLists.txt | 25 -
.../dataproxy-sdk-cpp/src/net/buffer_pool.cc | 428 ---------
.../dataproxy-sdk-cpp/src/net/buffer_pool.h | 119 ---
.../src/net/executor_thread_pool.cc | 120 ---
.../src/net/executor_thread_pool.h | 83 --
.../dataproxy-sdk-cpp/src/net/send_buffer.h | 153 ----
.../dataproxy-sdk-cpp/src/net/socket_connection.cc | 521 -----------
.../dataproxy-sdk-cpp/src/net/socket_connection.h | 126 ---
.../dataproxy-sdk-cpp/src/utils/capi_constant.h | 6 +-
.../dataproxy-sdk-cpp/src/utils/utils.cc | 125 ++-
.../dataproxy-sdk-cpp/src/utils/utils.h | 16 +-
.../dataproxy-sdk-cpp/test/CMakeLists.txt | 31 -
.../dataproxy-sdk-cpp/test/base/CMakeLists.txt | 24 -
.../test/base/client_config_test.cc | 75 --
.../dataproxy-sdk-cpp/test/base/pack_queue_test.cc | 129 ---
.../test/base/proxylist_config_test.cc | 147 ---
.../dataproxy-sdk-cpp/test/base/test_log.cc | 65 --
.../dataproxy-sdk-cpp/test/base/test_utils.cc | 204 -----
.../dataproxy-sdk-cpp/test/common.h | 83 --
.../dataproxy-sdk-cpp/test/main/CMakeLists.txt | 23 -
.../dataproxy-sdk-cpp/test/main/base_core_test.cc | 144 ---
.../dataproxy-sdk-cpp/test/main/init_exit_test.cc | 62 --
.../test/main/multi_thread_send.cc | 123 ---
.../dataproxy-sdk-cpp/test/main/random_send.cc | 114 ---
.../dataproxy-sdk-cpp/test/net/CMakeLists.txt | 25 -
.../dataproxy-sdk-cpp/test/net/big_pack_test.cc | 210 -----
.../dataproxy-sdk-cpp/test/net/buffer_pool_test.cc | 61 --
.../dataproxy-sdk-cpp/test/net/executor_test.cc | 171 ----
.../dataproxy-sdk-cpp/test/net/send_buffer_test.cc | 87 --
.../test/net/socket_connection_test.cc | 63 --
64 files changed, 314 insertions(+), 9361 deletions(-)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/CMakeLists.txt b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/CMakeLists.txt
index bc563f83b8..0fd025d96e 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/CMakeLists.txt
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/CMakeLists.txt
@@ -27,41 +27,40 @@ set(CMAKE_CXX_FLAGS "-std=c++11 -O2 -g -Wall -Wsign-compare -Wfloat-equal -fno-
include_directories(${PROJECT_SOURCE_DIR}/third_party/include)
include_directories(${PROJECT_SOURCE_DIR}/third_party/rapidjson/src/rapidjson/include)
include_directories(release/inc)
-include_directories(src/base)
-include_directories(src/net)
include_directories(src/utils)
include_directories(src/core)
include_directories(src/manager)
include_directories(src/group)
include_directories(src/protocol)
+include_directories(src/client)
link_directories(${PROJECT_SOURCE_DIR}/third_party/lib)
link_directories(${PROJECT_SOURCE_DIR}/third_party/lib64)
-# add_subdirectory(third_party)
-add_subdirectory(src/base)
-add_subdirectory(src/net)
+add_subdirectory(third_party)
add_subdirectory(src/utils)
add_subdirectory(src/config)
add_subdirectory(src/core)
add_subdirectory(src/manager)
add_subdirectory(src/group)
add_subdirectory(src/protocol)
-# add_subdirectory(test)
+add_subdirectory(src/client)
add_subdirectory(release)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/release/lib)
-# set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/release/so)
-aux_source_directory(src/base BASE_SRCS)
-aux_source_directory(src/net NET_SRCS)
-# dynamic library
-# add_library(dataproxy_sdk SHARED ${BASE_SRCS} ${NET_SRCS})
-# set_target_properties(dataproxy_sdk PROPERTIES PREFIX "")
-# target_link_libraries(dataproxy_sdk -llog4cplus -lsnappy -lcurl)
+aux_source_directory(src/utils UTILS)
+aux_source_directory(src/config CONFIGS)
+aux_source_directory(src/core CORE)
+aux_source_directory(src/manager MANAGER)
+aux_source_directory(src/group GROUP)
+aux_source_directory(src/protocol PROTOCOL)
+aux_source_directory(src/client CLIENT)
# static library
-add_library(dataproxy_sdk STATIC ${BASE_SRCS} ${NET_SRCS})
+add_library(dataproxy_sdk STATIC ${UTILS} ${CONFIGS} ${CORE} ${MANAGER} ${GROUP} ${PROTOCOL} ${CLIENT})
+
set_target_properties(dataproxy_sdk PROPERTIES OUTPUT_NAME "dataproxy_sdk" PREFIX "")
+
target_link_libraries(dataproxy_sdk liblog4cplus.a libsnappy.a libcurl.a)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/conf/config_example.json b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/conf/config_example.json
index 39353f8a51..c4ae2630de 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/conf/config_example.json
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/conf/config_example.json
@@ -24,7 +24,7 @@
"enable_hb": true,
"hb_interval": 60,
"proxy_update_interval": 10,
- "proxy_cfg_preurl": "http://127.0.0.1:8099/inlong/manager/openapi/dataproxy/getIpList",
+ "manager_url": "http://127.0.0.1:8099/inlong/manager/openapi/dataproxy/getIpList",
"need_auth": true,
"auth_id": "admin",
"auth_key": "87haw3VYTPqK5fK0"
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/demo/send_demo.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/demo/send_demo.cc
index 6a96ff9657..1c07fb99ac 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/demo/send_demo.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/demo/send_demo.cc
@@ -20,10 +20,10 @@
#include <chrono>
#include <iostream>
#include <string>
-#include <tc_api.h>
+#include "inlong_api.h"
using namespace std;
-using namespace dataproxy_sdk;
+using namespace inlong;
// user set call_back func
int call_back_func(const char *inlong_group_id, const char *inlong_stream_id, const char *msg, int msg_len, long report_time, const char *ip)
@@ -44,9 +44,9 @@ int main(int argc, char const *argv[])
cout << "USAGE: ./send_demo ../config/config_example.json" << endl;
return 0;
}
-
+ InLongApi inlong_api;
// step1. init
- if (tc_api_init(argv[1]))
+ if (inlong_api.InitApi(argv[1]))
{
cout << "init error" << endl;
return 0;
@@ -60,14 +60,14 @@ int main(int argc, char const *argv[])
inlong_group_id = argv[2];
inlong_stream_id = argv[3];
}
- cout << "inlong_group_id: "<<inlong_group_id<<"inlong_stream_id:"<<inlong_stream_id << endl;
+ cout << "inlong_group_id: "<<inlong_group_id<<",inlong_stream_id:"<<inlong_stream_id << endl;
string msg = "this is a test ttttttttttttttt; eiwhgreuhg jfdiowaehgorerlea; test end";
// step2. send
cout << "---->start tc_api_send" << endl;
for (size_t i = 0; i < count; i++)
{
- if (tc_api_send(inlong_group_id.c_str(), inlong_stream_id.c_str(), msg.c_str(), msg.length(), call_back_func))
+ if (inlong_api.Send(inlong_group_id.c_str(), inlong_stream_id.c_str(), msg.c_str(), msg.length(), call_back_func))
{
cout << "tc_api_send error;"
<< " ";
@@ -75,7 +75,7 @@ int main(int argc, char const *argv[])
}
// step3. close
- if (tc_api_close(1000))
+ if (inlong_api.CloseApi(1000))
{
cout << "close sdk error" << endl;
}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/client_config.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/client_config.h
deleted file mode 100644
index 81458227bb..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/client_config.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_CLIENT_CONFIG_H_
-#define DATAPROXY_SDK_BASE_CLIENT_CONFIG_H_
-
-#include <stdint.h>
-#include <string>
-#include <vector>
-
-namespace dataproxy_sdk
-{
- class ClientConfig
- {
- private:
- std::string config_path_;
- int32_t buf_num_; //sendbuf num of each bufpool, max_buf_pool_/(ext_pack_size_+400)
-
- public:
- //new parameters
- int32_t thread_nums_; //network thread nums
- int32_t shared_buf_nums_; //bufpool nums in shared method // TODO: cancel this, using buffer_num_per_groupId
- std::vector<std::string> inlong_group_ids_; // groupId set //TODO: using set instead of vector?
- bool enable_groupId_isolation_; // different groupId data are dispatched to different bufpool
- int32_t buffer_num_per_groupId_; // bufpool num of each groupId
- std::string net_tag_;
-
- //pack parameters
- bool enable_pack_;
- uint32_t pack_size_; //byte
- uint32_t pack_timeout_; //ms
- uint32_t ext_pack_size_; //byte, max length of msg
-
- //zip parameters
- bool enable_zip_;
- uint32_t min_zip_len_; //ms
-
- //resend parameters
- bool enable_retry_;
- uint32_t retry_interval_; //ms, resend interval if not receiving ack
- uint32_t retry_num_; //resend times
-
- //log parameters
- uint32_t log_num_;
- uint32_t log_size_; // M
- uint8_t log_level_; // trace(4)>debug(3)>info(2)>warn(1)>error(0)
- uint8_t log_file_type_; // output type:2->file, 1->console
- std::string log_path_;
- bool log_enable_limit_;
-
- // proxy parameters
- std::string proxy_URL_;
- bool enable_proxy_URL_from_cluster_;
-
- std::string proxy_cluster_URL_;
- uint32_t proxy_update_interval_; // minute
- uint32_t proxy_URL_timeout_; // second, request proxyList timeout
- uint32_t max_active_proxy_num_;
-
- //other parameters
- std::string ser_ip_; //local ip
- uint32_t max_buf_pool_; //byte, size of single bufpool
- uint32_t msg_type_;
- bool enable_TCP_nagle_;
- bool enable_heart_beat_;
- uint32_t heart_beat_interval_; //second
- bool enable_setaffinity_; // cpu setaffinity
- uint32_t mask_cpu_affinity_; // cpu setaffinity mask
- bool is_from_DC_; // sng_dc data
- uint16_t extend_field_;
-
- uint32_t buf_size_; // ext_pack_size+400;
-
- // auth settings
- bool need_auth_;
- std::string auth_id_;
- std::string auth_key_;
-
- void defaultInit();
-
- ClientConfig() { defaultInit(); }
-
- bool parseConfig(const std::string& config_path); // return false if parse failed
- void showClientConfig();
- void updateBufSize();
-
- inline bool enableCharGroupid() const { return (((extend_field_)&0x4) >> 2); } // use char type groupId、streadmId
- inline bool enableTraceIP() const { return (((extend_field_)&0x2) >> 1); }
- // datat type msg: datlen|data
- inline bool isNormalDataPackFormat() const { return ((5 == msg_type_) || ((msg_type_ >= 7) && (!(extend_field_ & 0x1)))); }
- // data&attr type msg: datlen|data|attr_len|attr
- inline bool isAttrDataPackFormat() const { return ((6 == msg_type_) || ((msg_type_ >= 7) && (extend_field_ & 0x1))); }
-
- inline int32_t bufNum() const { return buf_num_; }
- };
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_CLIENT_CONFIG_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/sdk_conf.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/sdk_conf.h
index b230f7a7bc..1beb05ea22 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/sdk_conf.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/sdk_conf.h
@@ -38,11 +38,12 @@ private:
void InitLogParam(const rapidjson::Value &doc);
void InitBusParam(const rapidjson::Value &doc);
void InitTcpParam(const rapidjson::Value &doc);
+ void InitAuthParm(const rapidjson::Value &doc);
void OthersParam(const rapidjson::Value &doc);
public:
// cache parameter
- std::vector<std::string> group_ids_; // Initialize the inlong groupid collection
+ std::vector<std::string> inlong_group_ids_; // Initialize the inlong groupid collection
uint32_t recv_buf_size_; // Receive buf size, tid granularity
uint32_t send_buf_size_; // Send buf size, bid granularity
@@ -81,6 +82,11 @@ public:
uint64_t tcp_idle_time_; // The time when tcpclient did not send data
uint32_t tcp_detection_interval_; // tcp-client detection interval
+ // auth settings
+ bool need_auth_;
+ std::string auth_id_;
+ std::string auth_key_;
+
// Other parameters
std::string local_ip_; // local ip
uint32_t local_port_ = 46801; // local port
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/tc_api.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/tc_api.h
deleted file mode 100644
index 8be16cba06..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/tc_api.h
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef DATAPROXY_SDK_RELEASE_INC_TC_API_H_
-#define DATAPROXY_SDK_RELEASE_INC_TC_API_H_
-
-#include <stdint.h>
-#include "user_msg.h"
-#include "client_config.h"
-
-namespace dataproxy_sdk
-{
-// send result
-enum SDKInvalidResult {
- kMultiInit = 4, // invoke tc_api_init repeatedly in the same process
- kErrorInit = 5, // tc_api_init error, user should check config file
- kMsgTooLong = 6, // msg len is more than ext_pack_size
- kInvalidInput = 7, // empty groupId or streamId or msg body
- kBufferPoolFull = 8, // sdk buffer is full, should adjust sending frequency, or increase max_buf_pool/thread_num in config file
- kFailGetConn = 9, // fail to get proxy connection, check whether groupId or proxy_cfg_preurl is correct
- kSendAfterClose = 10, // invoke send function after closing sdk
- kMultiExits = 11, // invoke tc_api_close function repeatedly
- kInvalidGroupId =12, // groupId is not set in config
- kFailGetBufferPool = 13,
- kFailGetSendBuf = 14,
- kFailWriteToBuf = 15,
- kErrorCURL = 16, // request manager error
- kErrorParseJson = 17,
- kFailGetPackQueue = 18, // failed to get pack queue
- kErrorAuthInfo = 19 // wrong authen
-
-};
-
-/**
- * @description: Before using sdk to send data, tc_api_init must be invoked
- * @return 0 if success
- * @param {char*} config_file, user configfile, prefer using absolute path
- */
-int32_t tc_api_init(const char* config_file);
-
-/**
- * @description: tc_api_init ext function
- * @return 0 if success
- * @param {char*} config_file - user configfile, prefer using absolute path
- * @param {int32_t} use_def - if use_def is zero, directly return if parsing config_file failed, which means this init failed
- */
-int32_t tc_api_init_ext(const char* config_file, int32_t use_def);
-
-/**
- * @description: Using ClientConfig to init api
- * @return 0 if success
- * @param {ClientConfig&} refer to client_config.h
- */
-int32_t tc_api_init(ClientConfig& client_config);
-
-
-/**
- * @description: send data
- * @return {*} 0 if success; non-zero means this send failed, refer to SDKInvalidResult above
- * @param {char*} inlong_group_id
- * @param {char*} inlong_stream_id
- * @param {char*} msg
- * @param {int32_t} msg_len
- * @param {UserCallBack} call_back
- */
-/*
- * UserCallBack function signature:
- * return: int32_t
- * parameters: (const char* inlong_group_id, const char* inlong_stream_id, const char* msg, int32_t msg_len, const int64_t report_time, const char* client_ip)
- */
-int32_t tc_api_send(const char* inlong_group_id, const char* inlong_stream_id, const char* msg, int32_t msg_len, UserCallBack call_back = NULL);
-
-int32_t tc_api_send_batch(const char* inlong_group_id, const char* inlong_stream_id, const char** msg_list, int32_t msg_cnt, UserCallBack call_back = NULL);
-
-
-/**
- * @description: send data, add msg_time based on tc_api_send
- * @return 0 if success
- */
-int32_t tc_api_send_ext(const char* inlong_group_id,
- const char* inlong_stream_id,
- const char* msg,
- int32_t msg_len,
- const int64_t msg_time,
- UserCallBack call_back = NULL);
-
-int32_t tc_api_send_base(const char* inlong_group_id,
- const char* inlong_stream_id,
- const char* msg,
- int32_t msg_len,
- const int64_t report_time,
- const char* client_ip,
- UserCallBack call_back = NULL);
-
-/**
- * @description: close sdk; if sdk is closed, you can't send data any more
- * @return 0 if success
- * @param {int32_t} max_waitms, millisecond, waiting data in memory to be sent
- */
-int32_t tc_api_close(int32_t max_waitms);
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_RELEASE_INC_TC_API_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/user_msg.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/user_msg.h
deleted file mode 100644
index 159483ce9d..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/user_msg.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_USER_MSG_H_
-#define DATAPROXY_SDK_BASE_USER_MSG_H_
-
-#include <memory>
-#include <stdint.h>
-#include <string>
-#include <functional>
-namespace dataproxy_sdk
-{
-// UserCallBack signature: inlong_group_id,inlong_stream_id,msg,msg_len, report_time, client_ip
-using UserCallBack = std::function<int32_t(const char*, const char*, const char*, int32_t, const int64_t, const char*)>;
-
-struct UserMsg
-{
- std::string msg;
- std::string client_ip;
- int64_t report_time;
- UserCallBack cb;
-
- int64_t user_report_time;
- std::string user_client_ip;
-
- std::string data_pack_format_attr; //"__addcol1__reptime=" + Utils::getFormatTime(data_time_) + "&__addcol2__ip=" + client_ip
- UserMsg(const std::string& mmsg,
- const std::string& mclient_ip,
- int64_t mreport_time,
- UserCallBack mcb,
- const std::string& attr,
- const std::string& u_ip,
- int64_t u_time)
- : msg(mmsg)
- , client_ip(mclient_ip)
- , report_time(mreport_time)
- , cb(mcb)
- , data_pack_format_attr(attr)
- , user_client_ip(u_ip)
- , user_report_time(u_time)
- {
- }
-};
-using UserMsgPtr = std::shared_ptr<UserMsg>;
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_USER_MSG_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/CMakeLists.txt b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/CMakeLists.txt
deleted file mode 100644
index 60c482e185..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/CMakeLists.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-cmake_minimum_required(VERSION 3.1)
-
-aux_source_directory(. BASE_SRCS)
-add_library(sdk_base STATIC ${BASE_SRCS})
-
-# set link lib
-target_link_libraries(sdk_base liblog4cplus.a libsnappy.a libcurl.a)
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/atomic.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/atomic.h
deleted file mode 100644
index 6babab6a03..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/atomic.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_ATOMIC_H_
-#define DATAPROXY_SDK_BASE_ATOMIC_H_
-
-#include <stdint.h>
-
-#include "noncopyable.h"
-namespace dataproxy_sdk {
-template <typename T>
-class AtomicIntegerT : noncopyable {
-private:
- volatile T value_;
-
-public:
- AtomicIntegerT() : value_(0) {}
-
- explicit AtomicIntegerT(T value) : value_(value) {}
-
- // if value_ equals oldval, update it as newval and return true;
- // otherwise, return false
- inline bool compareAndSwap(T oldval, T newval)
- {
- return __sync_bool_compare_and_swap(&value_, oldval, newval);
- }
-
- inline T get()
- {
- return __sync_val_compare_and_swap(&value_, 0, 0);
- }
-
- inline T getAndAdd(T x)
- {
- return __sync_fetch_and_add(&value_, x);
- }
-
- inline T getAndIncrease()
- {
- return __sync_fetch_and_add(&value_, 1);
- }
-
- inline T addAndGet(T x)
- {
- return getAndAdd(x) + x;
- }
-
- inline T incrementAndGet()
- {
- return addAndGet(1);
- }
-
- inline T decrementAndGet()
- {
- return addAndGet(-1);
- }
-
- inline void add(T x)
- {
- getAndAdd(x);
- }
-
- inline void increment()
- {
- incrementAndGet();
- }
-
- inline void decrement()
- {
- decrementAndGet();
- }
-
- inline T getAndSet(T newValue)
- {
- return __sync_lock_test_and_set(&value_, newValue);
- }
-};
-
-using AtomicInt = AtomicIntegerT<int32_t>;
-
-using AtomicUInt = AtomicIntegerT<uint32_t>;
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_ATOMIC_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/client_config.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/client_config.cc
deleted file mode 100644
index 678e20b769..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/client_config.cc
+++ /dev/null
@@ -1,705 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "client_config.h"
-
-#include <rapidjson/document.h>
-
-#include "sdk_constant.h"
-#include "logger.h"
-#include "utils.h"
-
-namespace dataproxy_sdk
-{
- bool ClientConfig::parseConfig(const std::string& config_path)
- {
- config_path_ = config_path;
- std::string file_content;
- if (!Utils::readFile(config_path_, file_content))
- {
- return false;
- }
-
- rapidjson::Document root;
-
- if (root.Parse(file_content.c_str()).HasParseError())
- {
- LOG_ERROR("failed to parse user config file: %s", config_path_.c_str());
- return false;
- }
-
- if (!root.HasMember("init-param"))
- {
- LOG_ERROR("param is not an object");
- return false;
- }
- const rapidjson::Value &doc = root["init-param"];
-
- // thread_num
- if (doc.HasMember("thread_num") && doc["thread_num"].IsInt() && doc["thread_num"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["thread_num"];
-
- thread_nums_ = obj.GetInt();
- LOG_WARN("thread_num in user config is: %d", thread_nums_);
- }
- else
- {
- thread_nums_ = constants::kThreadNums;
- LOG_WARN("thread_num in user config is not expect, then use default: %d", thread_nums_);
- }
- // shared_buf_nums
- if (doc.HasMember("shared_buf_num") && doc["shared_buf_num"].IsInt() && doc["shared_buf_num"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["shared_buf_num"];
-
- shared_buf_nums_ = obj.GetInt();
- LOG_WARN("shared_buf_num in user config is: %d", shared_buf_nums_);
- }
- else
- {
- shared_buf_nums_ = constants::kSharedBufferNums;
- LOG_WARN("shared_buf_num in user config is not expect, then use default: %d", shared_buf_nums_);
- }
-
- // inlong_group_ids, split by comma
- if (doc.HasMember("inlong_group_ids") && doc["inlong_group_ids"].IsString())
- {
- const rapidjson::Value &obj = doc["inlong_group_ids"];
- std::string groupIds_str = obj.GetString();
- int32_t size = Utils::splitOperate(groupIds_str, inlong_group_ids_, ",");
- LOG_WARN("inlong_group_ids in user config is: <%s>", Utils::getVectorStr(inlong_group_ids_).c_str());
- }
- else
- {
- LOG_WARN("inlong_group_ids in user config is empty");
- }
-
- // enable_groupId_isolation
- if (doc.HasMember("enable_groupId_isolation") && doc["enable_groupId_isolation"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_groupId_isolation"];
-
- enable_groupId_isolation_ = obj.GetBool();
- LOG_WARN("enable_groupId_isolation in user config is: %s", enable_groupId_isolation_ ? "true" : "false");
- }
- else
- {
- enable_groupId_isolation_ = constants::kEnableGroupidIsolation;
- LOG_WARN("enable_groupId_isolation in user config is not expect, then use default: %s", enable_groupId_isolation_ ? "true" : "false");
- }
-
- // if enable_groupId_isolation_ is true, groupIds cann't be empty
- if (enable_groupId_isolation_ && inlong_group_ids_.empty())
- {
- LOG_ERROR("inlong_group_ids is empty, check config!");
- return false;
- }
-
- // auth settings
- if (doc.HasMember("need_auth") && doc["need_auth"].IsBool() && doc["need_auth"].GetBool()){
- if (!doc.HasMember("auth_id") || !doc.HasMember("auth_key") || !doc["auth_id"].IsString() || !doc["auth_key"].IsString() ||doc["auth_id"].IsNull() || doc["auth_key"].IsNull())
- {
- LOG_ERROR("need_auth, but auth_id or auth_key is empty or not string type, check config!");
- return false;
- }
- need_auth_ = true;
- auth_id_ = doc["auth_id"].GetString();
- auth_key_ = doc["auth_key"].GetString();
- LOG_WARN("need_auth, auth_id:%s, auth_key:%s", auth_id_.c_str(), auth_key_.c_str());
- }
- else
- {
- need_auth_ = constants::kNeedAuth;
- LOG_WARN("need_auth is not expect, then use default:%s", need_auth_ ? "true" : "false");
- }
-
- // buffer_num_per_groupId
- if (doc.HasMember("buffer_num_per_groupId") && doc["buffer_num_per_groupId"].IsInt() && doc["buffer_num_per_groupId"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["buffer_num_per_groupId"];
-
- buffer_num_per_groupId_ = obj.GetInt();
- LOG_WARN("buffer_num_per_groupId in user config is: %d", buffer_num_per_groupId_);
- }
- else
- {
- buffer_num_per_groupId_ = constants::kBufferNumPerGroupid;
- LOG_WARN("buffer_num_per_groupId in user config is not expect, then use default: %d", buffer_num_per_groupId_);
- }
-
- // ser_ip
- if (doc.HasMember("ser_ip") && doc["ser_ip"].IsString())
- {
- const rapidjson::Value &obj = doc["ser_ip"];
-
- ser_ip_ = obj.GetString();
- LOG_WARN("ser_ip in user config is: %s", ser_ip_.c_str());
- }
- else
- {
- ser_ip_ = constants::kSerIP;
- LOG_WARN("ser_ip in user config is not expect, then use default: %s", ser_ip_.c_str());
- }
- // enable_pack
- if (doc.HasMember("enable_pack") && doc["enable_pack"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_pack"];
-
- enable_pack_ = obj.GetBool();
- LOG_WARN("enable_pack in user config is: %s", enable_pack_ ? "true" : "false");
- }
- else
- {
- enable_pack_ = constants::kEnablePack;
- LOG_WARN("enable_pack in user config is not expect, then use default: %s", enable_pack_ ? "true" : "false");
- }
- // pack_size
- if (doc.HasMember("pack_size") && doc["pack_size"].IsInt() && doc["pack_size"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["pack_size"];
-
- pack_size_ = obj.GetInt();
- LOG_WARN("pack_size in user config is: %d", pack_size_);
- }
- else
- {
- pack_size_ = constants::kPackSize;
- LOG_WARN("pack_size in user config is not expect, then use default: %d ms", pack_size_);
- }
- // pack_timeout
- if (doc.HasMember("pack_timeout") && doc["pack_timeout"].IsInt() && doc["pack_timeout"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["pack_timeout"];
-
- pack_timeout_ = obj.GetInt();
- LOG_WARN("pack_timeout in user config is: %d ms", pack_timeout_);
- }
- else
- {
- pack_timeout_ = constants::kPackTimeout;
- LOG_WARN("pack_timeout in user config is not expect, then use default: %d", pack_timeout_);
- }
- // ext_pack_size
- if (doc.HasMember("ext_pack_size") && doc["ext_pack_size"].IsInt() && doc["ext_pack_size"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["ext_pack_size"];
-
- ext_pack_size_ = obj.GetInt();
- LOG_WARN("ext_pack_size in user config is: %d", ext_pack_size_);
- }
- else
- {
- ext_pack_size_ = constants::kExtPackSize;
- LOG_WARN("ext_pack_size in user config is not expect, then use default: %d", ext_pack_size_);
- }
- // enable_zip
- if (doc.HasMember("enable_zip") && doc["enable_zip"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_zip"];
-
- enable_zip_ = obj.GetBool();
- LOG_WARN("enable_zip in user config is: %s", enable_zip_ ? "true" : "false");
- }
- else
- {
- enable_zip_ = constants::kEnablePack;
- LOG_WARN("enable_zip in user config is not expect, then use default: %s", enable_zip_ ? "true" : "false");
- }
- // min_zip_len
- if (doc.HasMember("min_zip_len") && doc["min_zip_len"].IsInt() && doc["min_zip_len"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["min_zip_len"];
-
- min_zip_len_ = obj.GetInt();
- LOG_WARN("min_zip_len in user config is: %d", min_zip_len_);
- }
- else
- {
- min_zip_len_ = constants::kMinZipLen;
- LOG_WARN("min_zip_len in user config is not expect, then use default: %d", min_zip_len_);
- }
- // enable_retry
- if (doc.HasMember("enable_retry") && doc["enable_retry"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_retry"];
-
- enable_retry_ = obj.GetBool();
- LOG_WARN("enable_retry in user config is: %s", enable_retry_ ? "true" : "false");
- }
- else
- {
- enable_retry_ = constants::kEnableRetry;
- LOG_WARN("enable_retry in user config is not expect, then use default: %s", enable_retry_ ? "true" : "false");
- }
- // retry_interval
- if (doc.HasMember("retry_ms") && doc["retry_ms"].IsInt() && doc["retry_ms"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["retry_ms"];
-
- retry_interval_ = obj.GetInt();
- LOG_WARN("retry_interval in user config is: %d ms", retry_interval_);
- }
- else
- {
- retry_interval_ = constants::kRetryInterval;
- LOG_WARN("retry_interval in user config is not expect, then use default: %d ms", retry_interval_);
- }
- // retry_num
- if (doc.HasMember("retry_num") && doc["retry_num"].IsInt() && doc["retry_num"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["retry_num"];
-
- retry_num_ = obj.GetInt();
- LOG_WARN("retry_num in user config is: %d times", retry_num_);
- }
- else
- {
- retry_num_ = constants::kRetryNum;
- LOG_WARN("retry_num in user config is not expect, then use default: %d times", retry_num_);
- }
- // log_num
- if (doc.HasMember("log_num") && doc["log_num"].IsInt() && doc["log_num"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["log_num"];
-
- log_num_ = obj.GetInt();
- LOG_WARN("log_num in user config is: %d", log_num_);
- }
- else
- {
- log_num_ = constants::kLogNum;
- LOG_WARN("log_num in user config is not expect, then use default: %d", log_num_);
- }
- // log_size
- if (doc.HasMember("log_size") && doc["log_size"].IsInt() && doc["log_size"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["log_size"];
-
- log_size_ = obj.GetInt();
- LOG_WARN("log_size in user config is: %dM", log_size_);
- }
- else
- {
- log_size_ = constants::kLogSize;
- LOG_WARN("log_size in user config is not expect, then use default: %dM", log_size_);
- }
- // log_level
- if (doc.HasMember("log_level") && doc["log_level"].IsInt() && doc["log_level"].GetInt() >= 0 && doc["log_level"].GetInt() <= 4)
- {
- const rapidjson::Value &obj = doc["log_level"];
-
- log_level_ = obj.GetInt();
- LOG_WARN("log_level in user config is: %d", log_level_);
- }
- else
- {
- log_level_ = constants::kLogLevel;
- LOG_WARN("log_level in user config is not expect, then use default: %d", log_level_);
- }
- // log_file_type
- if (doc.HasMember("log_file_type") && doc["log_file_type"].IsInt() && doc["log_file_type"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["log_file_type"];
-
- log_file_type_ = obj.GetInt();
- LOG_WARN("log_file_type in user config is: %d", log_file_type_);
- }
- else
- {
- log_file_type_ = constants::kLogFileType;
- LOG_WARN("log_file_type in user config is not expect, then use default: %d", log_file_type_);
- }
- // log_path
- if (doc.HasMember("log_path") && doc["log_path"].IsString())
- {
- const rapidjson::Value &obj = doc["log_path"];
-
- log_path_ = obj.GetString();
- LOG_WARN("log_path in user config is: %s", log_path_.c_str());
- }
- else
- {
- log_path_ = constants::kLogPath;
- LOG_WARN("log_path in user config is not expect, then use default: %s", log_path_.c_str());
- }
- // log_enable_limit
- if (doc.HasMember("log_enable_limit") && doc["log_enable_limit"].IsBool())
- {
- const rapidjson::Value &obj = doc["log_enable_limit"];
-
- log_enable_limit_ = obj.GetBool();
- LOG_WARN("log_enable_limit in user config is: %s", log_enable_limit_ ? "true" : "false");
- }
- else
- {
- log_enable_limit_ = constants::kLogEnableLimit;
- LOG_WARN("log_enable_limit in user config is not expect, then use default: %s", log_enable_limit_ ? "true" : "false");
- }
- // proxy_URL
- if (doc.HasMember("proxy_cfg_preurl") && doc["proxy_cfg_preurl"].IsString())
- {
- const rapidjson::Value &obj = doc["proxy_cfg_preurl"];
-
- proxy_URL_ = obj.GetString();
- LOG_WARN("proxy_cfg_preurl in user config is: %s", proxy_URL_.c_str());
- }
- else if (doc.HasMember("bus_cfg_preurl") && doc["bus_cfg_preurl"].IsString()) // compatible with internal usage
- {
- const rapidjson::Value &obj = doc["bus_cfg_preurl"];
-
- proxy_URL_ = obj.GetString();
- LOG_WARN("proxy_cfg_preurl in user config is: %s", proxy_URL_.c_str());
- }
-
- else
- {
- proxy_URL_ = constants::kProxyURL;
- LOG_WARN("proxy_cfg_url in user config is not expect, then use default: %s", proxy_URL_.c_str());
- }
- // proxy_cluster_URL_, only internal usage
- if (doc.HasMember("bus_cfg_url") && doc["bus_cfg_url"].IsString())
- {
- const rapidjson::Value &obj = doc["bus_cfg_url"];
-
- proxy_cluster_URL_ = obj.GetString();
- LOG_WARN("proxy_cluster_URL(proxy_cfg_url) in user config is: %s", proxy_cluster_URL_.c_str());
- }
- else
- {
- proxy_cluster_URL_ = constants::kProxyClusterURL;
- LOG_WARN("proxy_cluster_URL(proxy_cfg_url) in user config is not expect, then use default: %s", proxy_cluster_URL_.c_str());
- }
- // enable_proxy_URL_from_cluster
- if (doc.HasMember("enable_proxy_cfg_url") && doc["enable_proxy_cfg_url"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_proxy_cfg_url"];
-
- enable_proxy_URL_from_cluster_ = obj.GetBool();
- LOG_WARN("enable_proxy_URL_from_cluster in user config is: %s", enable_proxy_URL_from_cluster_ ? "true" : "false");
- }
- else
- {
- enable_proxy_URL_from_cluster_ = constants::kEnableProxyURLFromCluster;
- LOG_WARN("enable_proxy_URL_from_cluster in user config is not expect, then use default: %s", enable_proxy_URL_from_cluster_ ? "true" : "false");
- }
- // proxy_update_interval
- if (doc.HasMember("proxy_update_interval") && doc["proxy_update_interval"].IsInt() && doc["proxy_update_interval"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["proxy_update_interval"];
-
- proxy_update_interval_ = obj.GetInt();
- LOG_WARN("proxy_update_interval in user config is: %d minutes", proxy_update_interval_);
- }
- else if (doc.HasMember("bus_update_interval") && doc["bus_update_interval"].IsInt() && doc["bus_update_interval"].GetInt() > 0)//internal usage
- {
- const rapidjson::Value &obj = doc["bus_update_interval"];
-
- proxy_update_interval_ = obj.GetInt();
- LOG_WARN("proxy_update_interval in user config is: %d minutes", proxy_update_interval_);
- }
- else
- {
- proxy_update_interval_ = constants::kProxyUpdateInterval;
- LOG_WARN("proxy_update_interval in user config is not expect, then use default: %d minutes", proxy_update_interval_);
- }
- // proxy_URL_timeout
- if (doc.HasMember("proxy_url_timeout") && doc["proxy_url_timeout"].IsInt() && doc["proxy_url_timeout"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["proxy_url_timeout"];
-
- proxy_URL_timeout_ = obj.GetInt();
- LOG_WARN("proxy_url_timeout in user config is: %ds", proxy_URL_timeout_);
- }
- else if (doc.HasMember("bus_url_timeout") && doc["bus_url_timeout"].IsInt() && doc["bus_url_timeout"].GetInt() > 0) //internal usage
- {
- const rapidjson::Value &obj = doc["bus_url_timeout"];
-
- proxy_URL_timeout_ = obj.GetInt();
- LOG_WARN("proxy_url_timeout in user config is: %ds", proxy_URL_timeout_);
- }
- else
- {
- proxy_URL_timeout_ = constants::kProxyURLTimeout;
- LOG_WARN("proxy_url_timeout in user config is not expect, then use default: %ds", proxy_URL_timeout_);
- }
- // max_active_proxy_num
- if (doc.HasMember("max_active_proxy") && doc["max_active_proxy"].IsInt() && doc["max_active_proxy"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["max_active_proxy"];
-
- max_active_proxy_num_ = obj.GetInt();
- LOG_WARN("max_active_proxy in user config is: %d", max_active_proxy_num_);
- }
- else if (doc.HasMember("max_active_bus") && doc["max_active_bus"].IsInt() && doc["max_active_bus"].GetInt() > 0) //internal usage
- {
- const rapidjson::Value &obj = doc["max_active_bus"];
-
- max_active_proxy_num_ = obj.GetInt();
- LOG_WARN("max_active_proxy in user config is: %d", max_active_proxy_num_);
- }
- else
- {
- max_active_proxy_num_ = constants::kMaxActiveProxyNum;
- LOG_WARN("max_active_proxy in user config is not expect, then use default: %d", max_active_proxy_num_);
- }
- // max_buf_pool_
- if (doc.HasMember("max_buf_pool") && doc["max_buf_pool"].IsUint() && doc["max_buf_pool"].GetUint() > 0)
- {
- const rapidjson::Value &obj = doc["max_buf_pool"];
-
- max_buf_pool_ = obj.GetUint();
- LOG_WARN("max_buf_pool in user config is: %lu", max_buf_pool_);
- }
- else
- {
- max_buf_pool_ = constants::kMaxBufPool;
- LOG_WARN("max_buf_pool in user config is not expect, then use default: %lu", max_buf_pool_);
- }
- // msg_type
- if (doc.HasMember("msg_type") && doc["msg_type"].IsInt() && doc["msg_type"].GetInt() > 0 && doc["msg_type"].GetInt() < 9)
- {
- const rapidjson::Value &obj = doc["msg_type"];
-
- msg_type_ = obj.GetInt();
- LOG_WARN("msg_type in user config is: %d", msg_type_);
- }
- else
- {
- msg_type_ = constants::kMsgType;
- LOG_WARN("max_active_proxy in user config is not expect, then use default: %d", msg_type_);
- }
- // enable_TCP_nagle
- if (doc.HasMember("enable_tcp_nagle") && doc["enable_tcp_nagle"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_tcp_nagle"];
-
- enable_TCP_nagle_ = obj.GetBool();
- LOG_WARN("enable_tcp_nagle in user config is: %s", enable_TCP_nagle_ ? "true" : "false");
- }
- else
- {
- enable_TCP_nagle_ = constants::kEnableTCPNagle;
- LOG_WARN("enable_tcp_nagle in user config is not expect, then use default: %s", enable_TCP_nagle_ ? "true" : "false");
- }
- // enable_heart_beat
- if (doc.HasMember("enable_hb") && doc["enable_hb"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_hb"];
-
- enable_heart_beat_ = obj.GetBool();
- LOG_WARN("enable_hb in user config is: %s", enable_heart_beat_ ? "true" : "false");
- }
- else
- {
- enable_heart_beat_ = constants::kEnableHeartBeat;
- LOG_WARN("enable_hb in user config is not expect, then use default: %s", enable_heart_beat_ ? "true" : "false");
- }
-
- // heart_beat_interval
- if (doc.HasMember("hb_interval") && doc["hb_interval"].IsInt() && doc["hb_interval"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["hb_interval"];
-
- heart_beat_interval_ = obj.GetInt();
- LOG_WARN("heart_beat_interval in user config is: %ds", heart_beat_interval_);
- }
- else
- {
- heart_beat_interval_ = constants::kHeartBeatInterval;
- LOG_WARN("heart_beat_interval in user config is not expect, then use default: %ds", heart_beat_interval_);
- }
- // enable_setaffinity
- if (doc.HasMember("enable_setaffinity") && doc["enable_setaffinity"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_setaffinity"];
-
- enable_setaffinity_ = obj.GetBool();
- LOG_WARN("enable_setaffinity in user config is: %s", enable_setaffinity_ ? "true" : "false");
- }
- else
- {
- enable_setaffinity_ = constants::kEnableSetAffinity;
- LOG_WARN("enable_setaffinity in user config is not expect, then use default: %s", enable_setaffinity_ ? "true" : "false");
- }
- // mask_cpu_affinity
- if (doc.HasMember("mask_cpuaffinity") && doc["mask_cpuaffinity"].IsInt() && doc["mask_cpuaffinity"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["mask_cpuaffinity"];
-
- mask_cpu_affinity_ = obj.GetInt();
- LOG_WARN("mask_cpuaffinity in user config is: %d", mask_cpu_affinity_);
- }
- else
- {
- mask_cpu_affinity_ = constants::kMaskCPUAffinity;
- LOG_WARN("mask_cpuaffinity in user config is not expect, then use default: %d", mask_cpu_affinity_);
- }
- // is_from_DC
- if (doc.HasMember("enable_from_dc") && doc["enable_from_dc"].IsBool())
- {
- const rapidjson::Value &obj = doc["enable_from_dc"];
-
- is_from_DC_ = obj.GetBool();
- LOG_WARN("enable_from_dc in user config is: %s", is_from_DC_ ? "true" : "false");
- }
- else
- {
- is_from_DC_ = constants::kIsFromDC;
- LOG_WARN("enable_from_dc in user config is not expect, then use default: %s", is_from_DC_ ? "true" : "false");
- }
- // extend_field
- if (doc.HasMember("extend_field") && doc["extend_field"].IsInt() && doc["extend_field"].GetInt() > 0)
- {
- const rapidjson::Value &obj = doc["extend_field"];
-
- extend_field_ = obj.GetInt();
- LOG_WARN("extend_field in user config is: %d", extend_field_);
- }
- else
- {
- extend_field_ = constants::kExtendField;
- LOG_WARN("extend_field in user config is not expect, then use default: %d", extend_field_);
- }
-
- // net_tag
- if (doc.HasMember("net_tag") && doc["net_tag"].IsString())
- {
- const rapidjson::Value &obj = doc["net_tag"];
-
- net_tag_ = obj.GetString();
- LOG_WARN("net_tag in user config is: %s", net_tag_.c_str());
- }
- else
- {
- net_tag_ = constants::kNetTag;
- LOG_WARN("net_tag in user config is not expect, then use default: %s", net_tag_.c_str());
- }
-
- // set bufNum
- updateBufSize();
-
- LOG_WARN("sendBuf num of a pool is %d", buf_num_);
-
- return true;
- }
-
- void ClientConfig::updateBufSize()
- {
- buf_size_ = ext_pack_size_ + 400;
- buf_num_ = max_buf_pool_ / (buf_size_);
- }
-
- void ClientConfig::defaultInit(){
-
- thread_nums_=constants::kThreadNums;
- shared_buf_nums_=constants::kSharedBufferNums;
- enable_groupId_isolation_=constants::kEnableGroupidIsolation;
- buffer_num_per_groupId_=constants::kBufferNumPerGroupid;
- net_tag_=constants::kNetTag;
-
- enable_pack_=constants::kEnablePack;
- pack_size_=constants::kPackSize;
- pack_timeout_=constants::kPackTimeout;
- ext_pack_size_=constants::kExtPackSize;
-
- enable_zip_=constants::kEnableZip;
- min_zip_len_=constants::kMinZipLen;
-
- enable_retry_=constants::kEnableRetry;
- retry_interval_=constants::kRetryInterval;
- retry_num_=constants::kRetryNum;
-
- log_num_=constants::kLogNum;
- log_size_=constants::kLogSize;
- log_level_=constants::kLogLevel;
- log_file_type_=constants::kLogFileType;
- log_path_=constants::kLogPath;
- log_enable_limit_=constants::kLogEnableLimit;
-
- proxy_URL_=constants::kProxyURL;
- enable_proxy_URL_from_cluster_=constants::kEnableProxyURLFromCluster;
-
- proxy_cluster_URL_=constants::kProxyClusterURL;
- proxy_update_interval_=constants::kProxyUpdateInterval;
- proxy_URL_timeout_=constants::kProxyURLTimeout;
- max_active_proxy_num_=constants::kMaxActiveProxyNum;
-
- ser_ip_=constants::kSerIP;
- max_buf_pool_=constants::kMaxBufPool;
- msg_type_=constants::kMsgType;
- enable_TCP_nagle_=constants::kEnableTCPNagle;
- enable_heart_beat_=constants::kEnableHeartBeat;
- heart_beat_interval_=constants::kHeartBeatInterval;
- enable_setaffinity_=constants::kEnableSetAffinity;
- mask_cpu_affinity_=constants::kMaskCPUAffinity;
- is_from_DC_=constants::kIsFromDC;
- extend_field_=constants::kExtendField;
-
- need_auth_=constants::kNeedAuth;
-
- buf_size_ = ext_pack_size_ + 400;
- buf_num_ = max_buf_pool_ / (buf_size_);
- }
-
- void ClientConfig::showClientConfig()
- {
- LOG_WARN("thread_num: %d", thread_nums_);
- LOG_WARN("shared_buf_num: %d", shared_buf_nums_);
- LOG_WARN("inlong_group_ids: <%s>", Utils::getVectorStr(inlong_group_ids_).c_str());
- LOG_WARN("enable_groupId_isolation: %s", enable_groupId_isolation_ ? "true" : "fasle");
- LOG_WARN("buffer_num_per_groupId: %d", buffer_num_per_groupId_);
- LOG_WARN("ser_ip: %s", ser_ip_.c_str());
- LOG_WARN("enable_pack: %s", enable_pack_ ? "true" : "false");
- LOG_WARN("pack_size: %d", pack_size_);
- LOG_WARN("pack_timeout: %dms", pack_timeout_);
- LOG_WARN("ext_pack_size: %d", ext_pack_size_);
- LOG_WARN("enable_zip: %s", enable_zip_ ? "true" : "false");
- LOG_WARN("min_zip_len: %d", min_zip_len_);
- LOG_WARN("enable_retry: %s", enable_retry_ ? "true" : "false");
- LOG_WARN("retry_interval: %dms", retry_interval_);
- LOG_WARN("retry_num: %d times", retry_num_);
- LOG_WARN("log_num: %d", log_num_);
- LOG_WARN("log_size: %dM", log_size_);
- LOG_WARN("log_level: %d", log_level_);
- LOG_WARN("log_file_type: %d", log_file_type_);
- LOG_WARN("log_path: %s", log_path_.c_str());
- LOG_WARN("log_enable_limit: %s", log_enable_limit_ ? "true" : "false");
- LOG_WARN("proxy_cfg_preurl: %s", proxy_URL_.c_str());
- LOG_WARN("net_tag: %s", net_tag_.c_str());
- LOG_WARN("proxy_cluster_URL(proxy_cfg_url): %s", proxy_cluster_URL_.c_str());
- LOG_WARN("enable_proxy_URL_from_cluster: %s", enable_proxy_URL_from_cluster_ ? "true" : "false");
- LOG_WARN("proxy_update_interval: %d minutes", proxy_update_interval_);
- LOG_WARN("proxy_url_timeout: %ds", proxy_URL_timeout_);
- LOG_WARN("max_active_proxy: %d", max_active_proxy_num_);
- LOG_WARN("max_buf_pool: %lu", max_buf_pool_);
- LOG_WARN("msg_type: %d", msg_type_);
- LOG_WARN("enable_tcp_nagle: %s", enable_TCP_nagle_ ? "true" : "false");
- LOG_WARN("enable_hb: %s", enable_heart_beat_ ? "true" : "false");
- LOG_WARN("heart_beat_interval: %ds", heart_beat_interval_);
- LOG_WARN("enable_setaffinity: %s", enable_setaffinity_ ? "true" : "false");
- LOG_WARN("mask_cpuaffinity: %d", mask_cpu_affinity_);
- LOG_WARN("enable_from_dc: %s", is_from_DC_ ? "true" : "false");
- LOG_WARN("extend_field: %d", extend_field_);
- LOG_WARN("sendBuf num of a pool: %d", buf_num_);
- LOG_WARN("need_auth: %s", need_auth_ ? "true" : "false");
- LOG_WARN("auth_id: %s", auth_id_.c_str());
- LOG_WARN("auth_key: %s", auth_key_.c_str());
- }
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/ini_help.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/ini_help.cc
deleted file mode 100644
index 9959bafb7d..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/ini_help.cc
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "ini_help.h"
-
-#include <ctype.h>
-#include <fstream>
-#include <iostream>
-#include <stdio.h>
-#include <stdlib.h>
-
-namespace dataproxy_sdk
-{
-
-/**
- * @description: parse line content
- * @return {*} true if success
- * @param {string&} content
- * @param {string*} key
- * @param {string*} value
- */
-bool IniFile::parse(const std::string& content, std::string* key, std::string* value) { return split(content, "=", key, value); }
-
-int IniFile::updateSection(const std::string& cleanLine,
- const std::string& comment,
- const std::string& rightComment,
- IniSection** section)
-{
- IniSection* newSection;
- // find ']'
- size_t index = cleanLine.find_first_of(']');
- if (index == std::string::npos)
- {
- err_msg_ = std::string("no matched ] found");
- return 1;
- }
-
- int len = index - 1;
-
- if (len <= 0)
- {
- err_msg_ = std::string("section name is empty");
- return 1;
- }
-
- // set section name
- std::string s(cleanLine, 1, len);
-
- trim(s);
-
- //check section
- if (getSection(s) != NULL)
- {
- err_msg_ = std::string("section ") + s + std::string("already exist");
- return 1;
- }
-
- newSection = new IniSection();
- newSection->name = s;
- newSection->comment = comment;
- newSection->rightComment = rightComment;
-
- sections_.push_back(newSection);
-
- *section = newSection;
-
- return 0;
-}
-
-int IniFile::addKV(const std::string& cleanLine,
- const std::string& comment,
- const std::string& rightComment,
- IniSection* section)
-{
- std::string key, value;
-
- if (!parse(cleanLine, &key, &value))
- {
- err_msg_ = std::string("parse line failed:") + cleanLine;
- return 1;
- }
-
- Iterm item;
- item.key = key;
- item.value = value;
- item.comment = comment;
- item.rightComment = rightComment;
-
- section->items.push_back(item);
-
- return 0;
-}
-
-int IniFile::load(const std::string& fileName)
-{
- int err;
- std::string line;
- std::string comment;
- std::string rightComment;
- IniSection* currSection = NULL; //init a section
-
- close();
-
- ini_file_name_ = fileName;
- std::ifstream istream(ini_file_name_);
- if (!istream.is_open())
- {
- err_msg_ = std::string("open") + ini_file_name_ + std::string(" file failed");
- return 1;
- }
-
- // add new section
- currSection = new IniSection();
- currSection->name = "";
- sections_.push_back(currSection);
-
- // read line
- while (std::getline(istream, line))
- {
- trim(line);
-
- // skip empty
- if (line.length() <= 0)
- {
- comment += delim;
- continue;
- }
-
- // whether contains section or key
- // find '['
- if (line[0] == '[') { err = updateSection(line, comment, rightComment, &currSection); }
- else
- {
- err = addKV(line, comment, rightComment, currSection);
- }
-
- if (err != 0)
- {
- istream.close();
- return err;
- }
-
- // clear
- comment = "";
- rightComment = "";
- }
-
- istream.close();
-
- return 0;
-}
-
-IniSection* IniFile::getSection(const std::string& section)
-{
- for (SectionIterator it = sections_.begin(); it != sections_.end(); ++it)
- {
- if ((*it)->name == section) { return *it; }
- }
-
- return NULL;
-}
-
-int IniFile::getString(const std::string& section, const std::string& key, std::string* value)
-{
- return getValue(section, key, value);
-}
-
-int IniFile::getInt(const std::string& section, const std::string& key, int* intValue)
-{
- int err;
- std::string strValue;
-
- err = getValue(section, key, &strValue);
-
- *intValue = atoi(strValue.c_str());
-
- return err;
-}
-
-int IniFile::getValue(const std::string& section, const std::string& key, std::string* value)
-{
- std::string comment;
- return getValue(section, key, value, &comment);
-}
-
-int IniFile::getValue(const std::string& section, const std::string& key, std::string* value, std::string* comment)
-{
- IniSection* sect = getSection(section);
-
- if (sect == NULL)
- {
- err_msg_ = std::string("not find the section ") + section;
- return 1;
- }
-
- for (IniSection::ItermIterator it = sect->begin(); it != sect->end(); ++it)
- {
- if (it->key == key)
- {
- *value = it->value;
- *comment = it->comment;
- return 0;
- }
- }
-
- err_msg_ = std::string("not find the key ") + key;
- return 1;
-}
-
-void IniFile::close()
-{
- ini_file_name_ = "";
-
- for (SectionIterator it = sections_.begin(); it != sections_.end(); ++it)
- {
- delete (*it); // clear section
- }
-
- sections_.clear();
-}
-
-void IniFile::trim(std::string& str)
-{
- int len = str.length();
-
- int i = 0;
-
- while ((i < len) && isspace(str[i]) && (str[i] != '\0'))
- {
- i++;
- }
-
- if (i != 0) { str = std::string(str, i, len - i); }
-
- len = str.length();
-
- for (i = len - 1; i >= 0; --i)
- {
- if (!isspace(str[i])) { break; }
- }
-
- str = std::string(str, 0, i + 1);
-}
-
-bool IniFile::split(const std::string& str, const std::string& sep, std::string* pleft, std::string* pright)
-{
- size_t pos = str.find(sep);
- std::string left, right;
-
- if (pos != std::string::npos)
- {
- left = std::string(str, 0, pos);
- right = std::string(str, pos + 1);
-
- trim(left);
- trim(right);
-
- *pleft = left;
- *pright = right;
- return true;
- }
- else
- {
- left = str;
- right = "";
-
- trim(left);
-
- *pleft = left;
- *pright = right;
- return false;
- }
-}
-
-
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/ini_help.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/ini_help.h
deleted file mode 100644
index 235752652b..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/ini_help.h
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_INI_HELP_H_
-#define DATAPROXY_SDK_BASE_INI_HELP_H_
-
-#include <algorithm>
-#include <string.h>
-#include <string>
-#include <vector>
-
-namespace dataproxy_sdk
-{
-const char delim[] = "\n";
-struct Iterm
-{
- std::string key;
- std::string value;
- std::string comment;
- std::string rightComment;
-};
-
-struct IniSection
-{
- using ItermIterator = std::vector<Iterm>::iterator;
- ItermIterator begin()
- {
- return items.begin();
- }
-
- ItermIterator end()
- {
- return items.end();
- }
-
- std::string name;
- std::string comment;
- std::string rightComment;
- std::vector<Iterm> items;
-};
-
-class IniFile
-{
- public:
- IniFile(){}
- ~IniFile() { close(); }
-
- int load(const std::string& fileName);
- int getString(const std::string& section, const std::string& key, std::string* value);
- int getInt(const std::string& section, const std::string& key, int* value);
-
- private:
- IniSection* getSection(const std::string& section = "");
- static void trim(std::string& str);
- int updateSection(const std::string& cleanLine,
- const std::string& comment,
- const std::string& rightComment,
- IniSection** section);
- int addKV(const std::string& cleanLine,
- const std::string& comment,
- const std::string& rightComment,
- IniSection* section);
- void close();
- bool split(const std::string& str, const std::string& sep, std::string* left, std::string* right);
- bool parse(const std::string& content, std::string* key, std::string* value);
- int getValue(const std::string& section, const std::string& key, std::string* value);
- int getValue(const std::string& section, const std::string& key, std::string* value, std::string* comment);
-
- private:
- using SectionIterator = std::vector<IniSection*>::iterator;
- std::vector<IniSection*> sections_;
- std::string ini_file_name_;
- std::string err_msg_; // save err msg
-};
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_INI_HELP_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/logger.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/logger.cc
deleted file mode 100644
index 28e26483b7..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/logger.cc
+++ /dev/null
@@ -1,335 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "logger.h"
-
-#include <sys/time.h>
-#include <time.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <string.h>
-#include <pthread.h>
-#include <stdarg.h>
-#include <stddef.h>
-#include <unistd.h>
-#include <errno.h>
-
-namespace dataproxy_sdk
-{
- // log config
- debug_log_cfg gst_log_cfg;
-
- // log info
- debug_log_file gst_log_file;
-
- // log run info
- debug_log_run gst_log_run = {
- //.log_type =
- _LOG_PTY,
- //.log_change_min =
- 10,
- };
-
- pthread_mutex_t log_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-
- void debug_show_logcfg()
- {
- debug_log_cfg *log_cfg = &gst_log_cfg;
-
- LOG_ERROR("debug log config info. level(%d).", log_cfg->level);
-
- return;
- }
-
- void debug_get_date(char *logTime)
- {
- time_t mytime = time(NULL);
- struct tm curr;
- localtime_r(&mytime, &curr);
-
- if (curr.tm_year > 50)
- sprintf(logTime, "%04d-%02d-%02d %02d:%02d:%02d",
- curr.tm_year + 1900, curr.tm_mon + 1, curr.tm_mday,
- curr.tm_hour, curr.tm_min, curr.tm_sec);
- else
- sprintf(logTime, "%04d-%02d-%02d %02d:%02d:%02d",
- curr.tm_year + 2000, curr.tm_mon + 1, curr.tm_mday,
- curr.tm_hour, curr.tm_min, curr.tm_sec);
-
- return;
- }
-
- // update log file
- int32_t debug_shift_file(int32_t loglevel)
- {
- int32_t i = 0;
- int32_t ret = 0;
- char file_name[NAME_LEN];
- char new_name[NAME_LEN];
- struct stat file_stat;
- debug_log_file *log_file = &gst_log_file;
-
- memset(&file_name[0], 0x0, NAME_LEN);
- snprintf(&file_name[0], NAME_LEN - 1, "%s", &log_file->file_list[loglevel][0]);
-
- // get log file stat
- ret = stat(file_name, &file_stat);
- if (ret < 0)
- {
- printf("get log file:%s stat err.", file_name);
- return 1;
- }
-
- switch (log_file->shift_type)
- {
- case _LOG_SHIFT_SIZE:
- if (file_stat.st_size < log_file->max_size)
- {
- return 0;
- }
- break;
-
- case _LOG_SHIFT_COUNT:
- default:
- if (log_file->log_count < log_file->max_count)
- {
- return 0;
- }
- log_file->log_count = 0;
- break;
- }
-
- // create new log file
- for (i = log_file->max_lognum; i >= 0; i--)
- {
- if (i == 0)
- {
- snprintf(&file_name[0], NAME_LEN - 1, "%s",
- &log_file->file_list[loglevel][0]);
- }
- else
- {
- snprintf(&file_name[0], NAME_LEN - 1, "%s.%d",
- &log_file->file_list[loglevel][0], i);
- }
-
- if (0 == access(file_name, F_OK))
- {
- snprintf(&new_name[0], NAME_LEN - 1, "%s.%d",
- &log_file->file_list[loglevel][0], i + 1);
- if (rename(file_name, new_name) < 0)
- {
- printf("log file rename err(%d).", errno);
- return 1;
- }
- }
- }
-
- return 0;
- }
-
- // init log file
- int32_t debug_init_logfile(debug_log_file *log_file, debug_log_cfg *log_cfg)
- {
- const char *log_pre[_LOG_MAX_FILE] = {"error", "warn", "info", "debug", "trace", "stat"};
-
- memset(log_file, 0x0, sizeof(debug_log_file));
-
- for (int i = 0; i < _LOG_MAX_FILE; i++)
- {
- snprintf(&(log_file->file_list[i][0]), NAME_LEN - 1, "%ssdk_cpp_%s.log",
- &log_cfg->path[0], log_pre[i]);
- //printf("log will write to file:%s.\n", &log_file->file_list[i][0]);
- }
-
- log_file->shift_type = log_cfg->shift_type;
- log_file->max_lognum = log_cfg->num;
- log_file->max_size = log_cfg->size * 1024 * 1024;
- log_file->max_count = log_cfg->size * 1024 * 1024;
- log_file->log_count = log_cfg->size * 1024 * 1024;
-
- return 0;
- }
-
- static time_t last_write_log_time[_LOG_MAX_FILE] = {0};
- static int log_limit_cnt[_LOG_MAX_FILE] = {0};
-
- int32_t log_print(int32_t loglevel, int32_t log_time, char *format, ...)
- {
- va_list ap;
- char date[50];
- FILE *file_id;
- struct timeval tm;
- debug_log_file *log_file = &gst_log_file;
-
- if ((loglevel > (int32_t)SDK_CFG_LOG_LEVEL) && (loglevel != _LOG_STAT))
- {
- return 0;
- }
-
- #if 1
- time_t now = time(NULL);
- if (loglevel != _LOG_STAT && now - last_write_log_time[loglevel] < 10)
- {
- if (log_limit_cnt[loglevel] > 200)
- {
- return 0;
- }
-
- log_limit_cnt[loglevel]++;
- }
- else
- {
- last_write_log_time[loglevel] = now;
- log_limit_cnt[loglevel] = 0;
- }
- #endif
-
- debug_get_date(date);
- pthread_mutex_lock(&log_mutex);
- // open log file
- file_id = fopen(&(log_file->file_list[loglevel][0]), "a+");
- if (NULL == file_id)
- {
- printf("open file:%s err, errno:%d.\n", &(log_file->file_list[loglevel][0]), errno);
- pthread_mutex_unlock(&log_mutex);
- return 1;
- }
-
- va_start(ap, format);
- if (1 == log_time)
- {
- fprintf(file_id, "[%s]", date);
- }
- else
- {
- gettimeofday(&tm, NULL);
- fprintf(file_id, "[%s.%.6d]", date,
- (int32_t)tm.tv_usec);
- }
-
- vfprintf(file_id, format, ap);
- fprintf(file_id, "\n");
- va_end(ap);
- log_file->log_count++;
- fflush(file_id);
- fclose(file_id);
- debug_shift_file(loglevel);
- pthread_mutex_unlock(&log_mutex);
-
- return 0;
- }
-
- static int create_multi_dir(const char *path)
- {
- int i, len;
-
- len = strlen(path);
- char dir_path[len + 1];
- dir_path[len] = '\0';
-
- strncpy(dir_path, path, len);
-
- for (i = 0; i < len; i++)
- {
- if (dir_path[i] == '/' && i > 0)
- {
- dir_path[i] = '\0';
- if (access(dir_path, F_OK) < 0)
- {
- if (mkdir(dir_path, 0755) < 0)
- {
- printf("mkdir=%s:msg=%s\n", dir_path, strerror(errno));
- return -1;
- }
- }
- dir_path[i] = '/';
- }
- }
-
- return 0;
- }
-
- int32_t check_path(const char *path)
- {
- struct stat st_stat = {0};
- int ret = stat(path, &st_stat);
-
- if (ret && errno != ENOENT)
- {
- fprintf(stderr, "Check directory error: %s\n", strerror(errno));
- return 1;
- }
-
- if ((ret && errno == ENOENT) || (!ret && !S_ISDIR(st_stat.st_mode)))
- {
- // create dir, rwxr-xr-x
- if (mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH))
- {
- fprintf(stderr, "Crate directory error: %s\n", strerror(errno));
-
- return 1;
- }
- }
-
- return 0;
- }
-
- int32_t debug_init_log()
- {
- int32_t path_len = strlen(SDK_CFG_LOG_PATH);
- debug_log_cfg *log_cfg = &gst_log_cfg;
- debug_log_file *log_file = &gst_log_file;
- debug_log_run *log_run = &gst_log_run;
-
- log_cfg->level = SDK_CFG_LOG_LEVEL;
- log_cfg->num = SDK_CFG_LOG_NUM;
- log_cfg->size = SDK_CFG_LOG_SIZE;
- log_cfg->shift_type = _LOG_SHIFT_SIZE;
- log_cfg->file_type = 2;
-
- if (SDK_CFG_LOG_PATH[path_len] != '/')
- {
- snprintf(&log_cfg->path[0], NAME_LEN - 1,
- "%s%s", SDK_CFG_LOG_PATH, "/");
- }
- else
- {
- snprintf(&log_cfg->path[0], NAME_LEN - 1,
- "%s", SDK_CFG_LOG_PATH);
- }
-
- create_multi_dir(SDK_CFG_LOG_PATH);
-
- check_path(SDK_CFG_LOG_PATH);
-
- debug_init_logfile(log_file, log_cfg);
-
- log_run->log_pid = getpid();
- log_run->log_type = log_cfg->file_type;
- log_run->log_level = log_cfg->level;
-
- return 0;
- }
-
-
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/logger.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/logger.h
deleted file mode 100644
index 0aa10a406f..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/logger.h
+++ /dev/null
@@ -1,244 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_LOGGER_H_
-#define DATAPROXY_SDK_BASE_LOGGER_H_
-
-#include <stdint.h>
-#include <string.h>
-#include <string>
-#include <unistd.h>
-#include <sys/types.h>
-#include <vector>
-
-#include "sdk_constant.h"
-#include "sdk_core.h"
-
-namespace dataproxy_sdk
-{
- #ifndef SDK_CFG_LOG_LEVEL
- #define SDK_CFG_LOG_LEVEL g_config.log_level_
- #endif
-
- #ifndef SDK_CFG_LOG_NUM
- #define SDK_CFG_LOG_NUM g_config.log_num_
- #endif
-
- #ifndef SDK_CFG_LOG_SIZE
- #define SDK_CFG_LOG_SIZE g_config.log_size_
- #endif
-
- #ifndef SDK_CFG_LOG_PATH
- #define SDK_CFG_LOG_PATH g_config.log_path_.c_str()
- #endif
-
- #define NAME_LEN 512
-
- // only show fileName
- #define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
-
- enum _LOG_LEVEL_
- {
- _LOG_ERROR = 0x00,
- _LOG_WARN,
- _LOG_INFO,
- _LOG_DEBUG,
- _LOG_TRACE,
- _LOG_STAT,
- _LOG_MAX_FILE
- };
-
- enum _LOG_TYPE_
- {
- _LOG_PTY = 0x01,
- _LOG_FILE = 0x02,
- };
-
- enum _LOG_SHIFT_TYPE_
- {
- _LOG_SHIFT_SIZE = 0, /*shift by size*/
- _LOG_SHIFT_COUNT = 1, /*shift by log count*/
- };
-
- /* CFG debug info */
- enum SDK_LOG_DEBUG_EN {
- SDK_LOG_CHANGE_LEVEL,
- };
-
- typedef struct tag_debug_log_cfg
- {
- int32_t level;
- int32_t num;
- int32_t size;
- int32_t shift_type;
- int32_t file_type;
- char path[NAME_LEN];
- } debug_log_cfg;
-
- typedef struct tag_debug_log_file {
- char file_list[_LOG_MAX_FILE][NAME_LEN];
- int32_t shift_type;
- int32_t max_lognum;
- int32_t max_size;
- int32_t max_count;
- int32_t log_count;
- } debug_log_file;
-
- typedef struct tag_debug_log_run {
- pid_t log_pid;
- uint8_t log_type;
- uint8_t log_level;
- int32_t log_change_min;
- } debug_log_run;
-
- #define _log_pid_ gst_log_run.log_pid
- #define _log_type_ gst_log_run.log_type
- #define _log_level_ gst_log_run.log_level
-
-
- #define LOG_STAT(fmt, args...) \
- do{ \
- char log_time[50] = {0}; \
- if (_log_type_ & _LOG_FILE) \
- { \
- log_print(_LOG_STAT, 1, (char *)"PID[%d]STAT:%s:%.3d<%s>: "fmt,\
- _log_pid_, __FILENAME__, __LINE__, __FUNCTION__, ##args); \
- } \
- if (_log_type_ & _LOG_PTY) \
- { \
- debug_get_date(log_time); \
- fprintf(stderr, (char *)"[%s]PID[%d]STAT:%s:%.3d<%s>: "fmt"\n",\
- log_time, _log_pid_, __FILENAME__, __LINE__, \
- __FUNCTION__, ##args); \
- } \
- }while(0)
-
- #define LOG_ERROR(fmt, args...) \
- do{ \
- char log_time[50] = {0}; \
- if (_log_type_ & _LOG_FILE) \
- { \
- log_print(_LOG_ERROR, 1, (char *)"PID[%d]ERR:%s:%.3d<%s>: "fmt,\
- _log_pid_, __FILENAME__, __LINE__, __FUNCTION__, ##args); \
- } \
- if (_log_type_ & _LOG_PTY) \
- { \
- debug_get_date(log_time); \
- fprintf(stderr, (char *)"[%s]PID[%d]ERR:%s:%.3d<%s>: "fmt"\n", \
- log_time, _log_pid_, __FILENAME__, __LINE__, \
- __FUNCTION__, ##args); \
- } \
- }while(0)
-
- #define LOG_WARN(fmt, args...) \
- do{ \
- char log_time[50] = {0}; \
- if (_log_level_ < _LOG_WARN) \
- {break;} \
- if (_log_type_ & _LOG_FILE) \
- { \
- log_print(_LOG_WARN, 1, (char *)"PID[%d]WARN:%s:%.3d<%s>: "fmt,\
- _log_pid_, __FILENAME__, __LINE__, __FUNCTION__, ##args); \
- } \
- if (_log_type_ & _LOG_PTY) \
- { \
- debug_get_date(log_time); \
- fprintf(stderr, (char *)"[%s]PID[%d]WARN:%s:%.3d<%s>: "fmt"\n",\
- log_time, _log_pid_, __FILENAME__, __LINE__, \
- __FUNCTION__, ##args); \
- } \
- }while(0)
-
- #define LOG_INFO(fmt, args...) \
- do{ \
- char log_time[50] = {0}; \
- if (_log_level_ < _LOG_INFO) \
- {break;} \
- if (_log_type_ & _LOG_FILE) \
- { \
- log_print(_LOG_INFO, 1, (char *)"PID[%d]INFO:%s:%.3d<%s>: "fmt,\
- _log_pid_, __FILENAME__, __LINE__, __FUNCTION__, ##args); \
- } \
- if (_log_type_ & _LOG_PTY) \
- { \
- debug_get_date(log_time); \
- fprintf(stderr, (char *)"[%s]PID[%d]INFO:%s:%.3d<%s>: "fmt"\n",\
- log_time, _log_pid_, __FILENAME__, __LINE__, \
- __FUNCTION__, ##args); \
- } \
- }while(0)
-
- #define LOG_DEBUG(fmt, args...) \
- do{ \
- char log_time[50] = {0}; \
- if (_log_level_ < _LOG_DEBUG) \
- {break;} \
- if (_log_type_ & _LOG_FILE) \
- { \
- log_print(_LOG_DEBUG, 1, (char *)"PID[%d]DEBUG:%s:%.3d<%s>: "fmt,\
- _log_pid_, __FILENAME__, __LINE__, __FUNCTION__, ##args); \
- } \
- if (_log_type_ & _LOG_PTY) \
- { \
- debug_get_date(log_time); \
- fprintf(stderr, (char *)"[%s]PID[%d]DEBUG:%s:%.3d<%s>: "fmt"\n",\
- log_time, _log_pid_, __FILENAME__, __LINE__, \
- __FUNCTION__, ##args); \
- } \
- }while(0)
-
- #define LOG_TRACE(fmt, args...) \
- do{ \
- char log_time[50] = {0}; \
- if (_log_level_ < _LOG_TRACE) \
- {break;} \
- if (_log_type_ & _LOG_FILE) \
- { \
- log_print(_LOG_TRACE, 1, (char *)"PID[%d]TRACE:%s:%.3d<%s>: "fmt, \
- _log_pid_, __FILENAME__, __LINE__, __FUNCTION__, ##args); \
- } \
- if (_log_type_ & _LOG_PTY) \
- { \
- debug_get_date(log_time); \
- fprintf(stderr, (char *)"[%s]PID[%d]TRACE:%s:%.3d<%s>: "fmt"\n",\
- log_time, _log_pid_, __FILENAME__, __LINE__, \
- __FUNCTION__, ##args); \
- } \
- }while(0)
-
- #ifdef __cplusplus
- extern "C" {
- #endif
-
- extern debug_log_run gst_log_run;
- extern debug_log_cfg gst_log_cfg;
- extern debug_log_file gst_log_file;
- extern void debug_get_date(char *logTime);
-
- int32_t log_print(int32_t log_level, int32_t log_time, char *format, ...);
- int32_t debug_init_log();
- void tc_log_level_restore(void);
-
- #ifdef __cplusplus
- }
- #endif
-
-} // namespace dataproxy_sdk
-
-#endif // CPAI_BASE_LOGGER_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/msg_protocol.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/msg_protocol.h
deleted file mode 100644
index 8769683824..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/msg_protocol.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_MSG_PROTOCOL_H_
-#define DATAPROXY_SDK_BASE_MSG_PROTOCOL_H_
-
-#include "send_buffer.h"
-#include <memory>
-#include <stdint.h>
-#include <string>
-#include <vector>
-
-namespace dataproxy_sdk
-{
-// msg_type 1~6
-// totalLen(4)|msgtype(1)|bodyLen(4)|body(x)|attrLen(4)|attr(attr_len)
-#pragma pack(1)
-struct ProtocolMsgHead
-{
- uint32_t total_len;
- char msg_type;
-};
-struct ProtocolMsgBody
-{
- uint32_t body_len;
- char body[0];
-};
-struct ProtocolMsgTail
-{
- uint32_t attr_len;
- char attr[0];
-};
-// msg_type=7
-// totalLen(4)|msgtype(1)|groupid_num(2)|streamid_num(2)|ext_field(2)|data_time(4)|cnt(2)|uniq(4)|bodyLen(4)|body(x)|attrLen(4)|attr(attr_len)|magic(2)
-struct BinaryMsgHead
-{
- uint32_t total_len;
- char msg_type;
- uint16_t groupid_num;
- uint16_t streamid_num;
- uint16_t ext_field;
- uint32_t data_time; //second, last pack time
- uint16_t cnt;
- uint32_t uniq;
-};
-// msg_type7 ack pack
-struct BinaryMsgAck
-{
- uint32_t total_len;
- char msg_type;
- uint32_t uniq; //buffer id
- uint16_t attr_len;
- char attr[0];
- uint16_t magic;
-};
-
-// binary hb and hb ack, msg_type=8; hb ack: body_ver is 1, if body_len is 2, there is load in body
-struct BinaryHB
-{
- uint32_t total_len;
- char msg_type;
- uint32_t data_time; // second
- uint8_t body_ver; // body_ver=1
- uint32_t body_len; // body_len=0
- char body[0];
- uint16_t attr_len; // attr_len=0;
- char attr[0];
- uint16_t magic;
-};
-#pragma pack()
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_MSG_PROTOCOL_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/noncopyable.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/noncopyable.h
deleted file mode 100644
index e3569f1eef..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/noncopyable.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_NONCOPYABLE_H_
-#define DATAPROXY_SDK_BASE_NONCOPYABLE_H_
-
-namespace dataproxy_sdk
-{
-class noncopyable
-{
- public:
- noncopyable(const noncopyable&) = delete;
- void operator=(const noncopyable&) = delete;
-
- protected:
- noncopyable() = default;
- ~noncopyable() = default;
-};
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_NONCOPYABLE_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/pack_queue.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/pack_queue.cc
deleted file mode 100644
index 9a575f1f6a..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/pack_queue.cc
+++ /dev/null
@@ -1,580 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "pack_queue.h"
-
-#include <cstdlib>
-#include <functional>
-
-#include "proxylist_config.h"
-#include "sdk_constant.h"
-#include "sdk_core.h"
-#include "logger.h"
-#include "msg_protocol.h"
-#include "send_buffer.h"
-#include "socket_connection.h"
-#include "tc_api.h"
-#include "utils.h"
-#include "msg_protocol.h"
-
-namespace dataproxy_sdk
-{
- PackQueue::PackQueue(const std::string &inlong_group_id, const std::string &inlong_stream_id)
- : cur_len_(0), inlong_group_id_(inlong_group_id), inlong_stream_id_(inlong_stream_id), groupId_num_(0), streamId_num_(0), msg_type_(g_config.msg_type_), data_capacity_(g_config.buf_size_)
- {
- data_ = new char[data_capacity_];
- memset(data_, 0x0, data_capacity_);
- topic_desc_ = "groupId=" + inlong_group_id_ + "&streamId=" + inlong_stream_id_;
- first_use_ = Utils::getCurrentMsTime();
- last_use_ = Utils::getCurrentMsTime();
- data_time_ = 0;
- }
-
- PackQueue::~PackQueue()
- {
- if (data_)
- {
- delete[] data_;
- data_ = nullptr;
- }
- }
-
- int32_t PackQueue::sendMsg(const std::string &msg,
- const std::string &inlong_group_id,
- const std::string &inlong_stream_id,
- const std::string &client_ip,
- uint64_t report_time,
- UserCallBack call_back)
- {
- std::lock_guard<std::mutex> lck(mutex_);
-
- //pack previous
- if (isTriggerPack(report_time, msg.size()))
- {
- int32_t res = writeToBuf();
- if (res)
- {
- increasePackErr();
- return res;
- }
- }
-
- //write data to packqueue
- int32_t append_ret = appendMsg(msg, client_ip, report_time, call_back);
- if (append_ret)
- {
- LOG_ERROR("fail to write to pack queue, inlong_group_id: %s, inlong_stream_id: %s", inlong_group_id.c_str(), inlong_stream_id.c_str());
- return append_ret;
- }
-
- //if uneable_pack, single msg is written to packqueue and directly sent to buf
- if (!g_config.enable_pack_)
- {
- int32_t res = writeToBuf();
- if (res)
- {
- increasePackErr();
- return res;
- }
- }
- return 0;
- }
-
- //send pack data to buf
- int32_t PackQueue::writeToBuf()
- {
- if (inlong_group_id_.empty())
- {
- LOG_ERROR("something is wrong, check!!");
- return SDKInvalidResult::kFailGetConn;
- }
- if (msg_set_.empty())
- {
- LOG_ERROR("no msg in msg_set, check!");
- return SDKInvalidResult::kFailGetPackQueue;
- }
- auto conn = g_clusters->getSendConn(inlong_group_id_);
- if (!conn)
- {
- LOG_ERROR("no avaliable connection for inlong_group_id: %s, try later", inlong_group_id_.c_str());
- return SDKInvalidResult::kFailGetConn;
- }
- auto pool = g_pools->getPool(inlong_group_id_);
- if (!pool)
- {
- return SDKInvalidResult::kFailGetBufferPool;
- }
-
- SendBuffer *send_buf = nullptr;
-
- int32_t res = pool->getSendBuf(send_buf);
- if (res)
- {
- return res;
- }
- if (!send_buf)
- {
- LOG_ERROR("failed to get send_buf, something gets wrong, checkout!");
- return SDKInvalidResult::kFailGetSendBuf;
- }
-
- //lock sendbuf and write pack data to sendbuf
- {
- std::lock_guard<std::mutex> buf_lck(send_buf->mutex_);
-
- uint32_t len = 0;
- int32_t msg_cnt = msg_set_.size();
- // std::string msg_groupid = inlong_group_id_;
- uint32_t uniq_id = g_send_msgid.incrementAndGet();
- if (!packOperate(send_buf->content(), len, uniq_id) || len == 0)
- {
- LOG_ERROR("failed to write data to send buf from pack queue, pool id:%d, buf id:%d", pool->poolId(), pool->writeId());
- return SDKInvalidResult::kFailWriteToBuf;
- }
- send_buf->setLen(len);
- send_buf->setMsgCnt(msg_cnt);
- send_buf->setGroupid(inlong_group_id_);
- send_buf->setStreamid(inlong_stream_id_);
- send_buf->setUniqId(uniq_id);
- send_buf->setTarget(conn);
- send_buf->setIsPacked(true);
-
- for (auto it : msg_set_)
- {
- send_buf->addUserMsg(it);
- }
- }
-
- pack_num_.increment();
- g_pools->addUid2BufPool(send_buf->uniqId(),pool);//used for ack finding
-
- resetPackQueue();
-
- pool->sendBufToConn(send_buf);
- return 0;
- }
-
- int32_t PackQueue::appendMsg(const std::string &msg, std::string client_ip, int64_t report_time, UserCallBack call_back)
- {
- //too long msg
- if (msg.size() > g_config.ext_pack_size_)
- {
- LOG_ERROR("msg len (%d) more than ext_pack_size (%d)", msg.size(), g_config.ext_pack_size_);
- return SDKInvalidResult::kMsgTooLong;
- }
-
- //if datatime is illegal, fix it using current time
- if (Utils::isLegalTime(report_time))
- data_time_ = report_time;
- else
- {
- data_time_ = Utils::getCurrentMsTime();
- pack_redotime_cnt_.increment();
- }
-
- //used for callback
- if (call_back)
- {
- std::string user_client_ip = client_ip;
- int64_t user_report_time = report_time;
- if (client_ip.empty())
- {
- client_ip = "127.0.0.1";
- }
- std::string data_pack_format_attr = "__addcol1__reptime=" + Utils::getFormatTime(data_time_) + "&__addcol2__ip=" + client_ip;
- msg_set_.emplace_back(std::make_shared<UserMsg>(msg, client_ip, data_time_, call_back, data_pack_format_attr, user_client_ip, user_report_time));
- }
-
- cur_len_ += msg.size() + 1; // '\n' using one byte
-
- if (g_config.isNormalDataPackFormat())
- {
- cur_len_ += 4;
- }
- if (g_config.isAttrDataPackFormat())
- {
- cur_len_ += constants::kAttrLen + 8;
- }
-
- //update last using time
- last_use_ = Utils::getCurrentMsTime();
-
- return 0;
- }
-
- /**
- * @description: whether trigger pack data
- * @param {uint64_t} report_time
- * @param {int32_t} msg_len
- */
- bool PackQueue::isTriggerPack(uint64_t report_time, int32_t msg_len)
- {
- if (0 == cur_len_ || msg_set_.empty())
- return false;
-
- if (!Utils::isLegalTime(report_time))
- {
- report_time = Utils::getCurrentMsTime();
- }
-
- int64_t max_pack_time_interval = 1800000; // FIXME: use user config？
- bool time_trigger = false; //timeout trigger
- bool len_trigger = false; //content trigger
- if (llabs(report_time - data_time_) > max_pack_time_interval ||
- (report_time != data_time_ && report_time / 1000 / 3600 != data_time_ / 1000 / 3600))
- {
- time_trigger = true;
- }
- if (msg_len + cur_len_ > g_config.pack_size_)
- {
- len_trigger = true;
- }
-
- return (time_trigger || len_trigger);
- }
-
- /**
- * @description: do pack operate
- * @param {int8*} pack_data: packed binary data
- * @param {uint32_t&} out_len
- * @return {*} true if pack successfully
- */
- bool PackQueue::packOperate(char *pack_data, uint32_t &out_len, uint32_t uniq_id)
- {
- if (!pack_data)
- {
- LOG_ERROR("nullptr, failed to allocate memory for buf");
- return false;
- }
- //add body into data_, then zip and copy to buffer
- uint32_t idx = 0;
- for (auto &it : msg_set_)
- {
- //msg>=5,body format: data_len|data
-
- if (msg_type_ >= 5) //add data_len
- {
- *(uint32_t *)(&data_[idx]) = htonl(it->msg.size());
- idx += sizeof(uint32_t);
- }
- //add data
- memcpy(&data_[idx], it->msg.data(), it->msg.size());
- idx += static_cast<uint32_t>(it->msg.size());
-
- //add attrlen|attr
- if (g_config.isAttrDataPackFormat())
- {
- *(uint32_t *)(&data_[idx]) = htonl(it->data_pack_format_attr.size());
- idx += sizeof(uint32_t);
-
- memcpy(&data_[idx], it->data_pack_format_attr.data(), it->data_pack_format_attr.size());
- idx += static_cast<uint32_t>(it->data_pack_format_attr.size());
- }
-
- // msgtype = 2/3 support '\n'
- if (msg_type_ == 2 || msg_type_ == 3)
- {
- data_[idx] = '\n';
- ++idx;
- }
- }
-
- //preprocess attr
- uint32_t cnt = 1;
- if (msg_set_.size())
- {
- cnt = msg_set_.size();
- }
-
- //pack
- if (msg_type_ >= constants::kBinPackMethod)
- {
- char *bodyBegin = pack_data + sizeof(BinaryMsgHead) + sizeof(uint32_t); //head+body_len
- uint32_t body_len = 0;
-
- std::string snappy_res;
- bool isSnappy = isZipAndOperate(snappy_res, idx);
- char real_msg_type;
-
- if (isSnappy) // need zip
- {
- body_len = static_cast<uint32_t>(snappy_res.size());
- memcpy(bodyBegin, snappy_res.data(), body_len); //copy data to buf
- // msg_type
- real_msg_type = (msg_type_ | constants::kBinSnappyFlag);
- }
- else
- {
- body_len = idx;
- memcpy(bodyBegin, data_, body_len);
- real_msg_type = msg_type_;
- }
- *(uint32_t *)(&(pack_data[sizeof(BinaryMsgHead)])) = htonl(body_len); //set bodylen
-
- bodyBegin += body_len;
-
- // groupid_num、streamid_num、ext_field、data_time、cnt、uniq
- uint32_t char_groupid_flag = 0;
- std::string groupid_streamid_char;
- uint16_t groupid_num = 0, streamid_num = 0;
- if (g_config.enableCharGroupid() || groupId_num_ == 0 || streamId_num_ == 0) //using string groupid and streamid
- {
- groupid_num = 0;
- streamid_num = 0;
- groupid_streamid_char = topic_desc_;
- char_groupid_flag = 0x4;
- }
- else
- {
- groupid_num = groupId_num_;
- streamid_num = streamId_num_;
- }
- uint16_t ext_field = (g_config.extend_field_ | char_groupid_flag);
- uint32_t data_time = data_time_ / 1000;
-
- // attr
- std::string attr;
- if (g_config.enableTraceIP())
- {
- if (groupid_streamid_char.empty())
- attr = "node1ip=" + g_config.ser_ip_ + "&rtime1=" + std::to_string(Utils::getCurrentMsTime());
- else
- attr = groupid_streamid_char + "&node1ip=" + g_config.ser_ip_ + "&rtime1=" + std::to_string(Utils::getCurrentMsTime());
- }
- else
- {
- attr = topic_desc_;
- }
- // attrlen
- *(uint16_t *)bodyBegin = htons(attr.size());
- bodyBegin += sizeof(uint16_t);
- // attr
- memcpy(bodyBegin, attr.data(), attr.size());
- bodyBegin += attr.size();
-
- // magic
- *(uint16_t *)bodyBegin = htons(constants::kBinaryMagic);
-
- uint32_t total_len = 25 + body_len + attr.size();
-
- // header
- char *p = pack_data;
- *(uint32_t *)p = htonl(total_len);
- p += 4;
- *p = real_msg_type;
- ++p;
- *(uint16_t *)p = htons(groupid_num);
- p += 2;
- *(uint16_t *)p = htons(streamid_num);
- p += 2;
- *(uint16_t *)p = htons(ext_field);
- p += 2;
- *(uint32_t *)p = htonl(data_time);
- p += 4;
- *(uint16_t *)p = htons(cnt);
- p += 2;
- *(uint32_t *)p = htonl(uniq_id);
-
- out_len = total_len + 4;
- }
- else
- {
- if (msg_type_ == 3 || msg_type_ == 2)
- {
- --idx;
- }
-
- // body whether needs zip
- char *bodyBegin = pack_data + sizeof(ProtocolMsgHead) + sizeof(uint32_t);
- uint32_t body_len = 0;
- std::string snappy_res;
- bool isSnappy = isZipAndOperate(snappy_res, idx);
- if (isSnappy)
- {
- body_len = static_cast<uint32_t>(snappy_res.size());
- memcpy(bodyBegin, snappy_res.data(), body_len); //copy
- }
- else
- {
- body_len = idx;
- memcpy(bodyBegin, data_, body_len);
- }
- *(uint32_t *)(&(pack_data[sizeof(ProtocolMsgHead)])) = htonl(body_len); //set bodylen
- bodyBegin += body_len;
-
- // attr
- std::string attr;
- attr = topic_desc_;
- attr += "&dt=" + std::to_string(data_time_);
- attr += "&mid=" + std::to_string(uniq_id);
- if (isSnappy)
- attr += "&cp=snappy";
- attr += "&cnt=" + std::to_string(cnt);
- attr += "&sid=" + std::string(Utils::getSnowflakeId());
- if (g_config.is_from_DC_)
- { //&__addcol1_reptime=yyyymmddHHMMSS&__addcol2__ip=BBB&f=dc
- attr += "&__addcol1_reptime=" + Utils::getFormatTime(Utils::getCurrentMsTime()) + "&__addcol2__ip=" + g_config.ser_ip_ + "&f=dc";
- }
-
- // attrlen
- *(uint32_t *)bodyBegin = htonl(attr.size());
- bodyBegin += sizeof(uint32_t);
- // attr
- memcpy(bodyBegin, attr.data(), attr.size());
-
- // total_len
- uint32_t total_len = 1 + 4 + body_len + 4 + attr.size();
- *(uint32_t *)pack_data = htonl(total_len);
- // msg_type
- *(&pack_data[4]) = msg_type_;
-
- LOG_TRACE("after packoperate: total_len:%d, body_len:%d, attr_len:%d", total_len, body_len, attr.size());
-
- out_len = total_len + 4;
- }
- return true;
- }
-
- /**
- * @description: whether body data should be zipped; if needs, zip data and save as res
- * @param {string&} res
- * @param {uint32_t} real_cur_len: data len before zip
- * @return {*} true if needing zip
- */
- bool PackQueue::isZipAndOperate(std::string &res, uint32_t real_cur_len)
- {
- if (g_config.enable_zip_ && real_cur_len > g_config.min_zip_len_)
- {
- LOG_TRACE("start snappy.");
- Utils::zipData(data_, real_cur_len, res);
- return true;
- }
- else
- return false;
- }
-
- void PackQueue::checkQueue(bool isLastPack)
- {
- if (cur_len_ == 0 || msg_set_.empty())
- return;
- //no timeout, and it isn't last packing
- if (Utils::getCurrentMsTime() - first_use_ < g_config.pack_timeout_ && !isLastPack)// FIXME:should use first_use instead of last_use?
- return;
- LOG_TRACE("start auto pack, inlong_group_id:%s, inlong_stream_id:%s", inlong_group_id_.c_str(), inlong_stream_id_.c_str());
-
- std::lock_guard<std::mutex> lck(mutex_);
- last_use_ = Utils::getCurrentMsTime();
- //send fail, callback and reset
- if (writeToBuf())
- {
- for (auto& it : msg_set_)
- {
- if (it->cb) { it->cb(inlong_group_id_.data(), inlong_stream_id_.data(), it->msg.data(), it->msg.size(), it->user_report_time, it->user_client_ip.data()); }
-
- }
- resetPackQueue();
-
- }
- }
-
- GlobalQueues::~GlobalQueues()
- {
- closeCheckSubroutine();
- if (worker_.joinable())
- {
- worker_.join();
- }
- for (auto it : queues_)
- {
- it.second->checkQueue(true);
- }
- queues_.clear();
- }
-
- void GlobalQueues::startCheckSubroutine() { worker_ = std::thread(&GlobalQueues::checkPackQueueSubroutine, this); }
-
- void GlobalQueues::checkPackQueueSubroutine()
- {
- LOG_INFO("start checkPackQueue subroutine");
- while (!exit_flag_)
- {
- Utils::taskWaitTime(2); // FIXME:improve pack interval
- for (auto it : queues_)
- {
- it.second->checkQueue(exit_flag_);
- }
- }
- LOG_WARN("exit checkPackQueue subroutine");
- }
-
- //get pack queue, create if not exists
- PackQueuePtr GlobalQueues::getPackQueue(const std::string &inlong_group_id, const std::string &inlong_stream_id)
- {
- std::lock_guard<std::mutex> lck(mutex_);
-
- auto it = queues_.find(inlong_group_id + inlong_stream_id);
- if (it != queues_.end())
- return it->second;
- else
- {
- PackQueuePtr p = std::make_shared<PackQueue>(inlong_group_id, inlong_stream_id);
- queues_.emplace(inlong_group_id + inlong_stream_id, p);
- return p;
- }
- }
- void GlobalQueues::printAck()
- {
- if (queues_.empty())
- return;
- for (auto &it : queues_)
- {
- LOG_STAT("dataproxy_sdk_cpp #local:%s#%s#success send msg:%d", g_config.ser_ip_.c_str(), it.second->topicDesc().c_str(),
- it.second->success_num_.getAndSet(0));
- }
- }
-
- void GlobalQueues::printTotalAck()
- {
- if (queues_.empty())
- return;
- for (auto &it : queues_)
- {
- LOG_STAT("dataproxy_sdk_cpp #local:%s#%s#total success msg:%d", g_config.ser_ip_.c_str(), it.second->topicDesc().c_str(),
- it.second->total_success_num_.get());
- }
- }
-
- void GlobalQueues::showState()
- {
- if (1 == user_exit_flag.get())
- return;
- uint32_t total_pack = 0;
- for (auto &it : queues_)
- {
- uint32_t pack = it.second->pack_num_.getAndSet(0);
- total_pack += pack;
- LOG_DEBUG("toipc:%s, pack_num:%d", it.second->topicDesc().c_str(), pack);
- }
- LOG_DEBUG("total_pack:%d", total_pack);
-
- g_pools->showState();
- g_executors->showState();
- }
-
-} // namespace dataproxy_sdk
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/pack_queue.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/pack_queue.h
deleted file mode 100644
index 4debb899d5..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/pack_queue.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_PACK_QUEUE_H_
-#define DATAPROXY_SDK_BASE_PACK_QUEUE_H_
-
-#include <memory>
-#include <mutex>
-#include <string>
-#include <thread>
-#include <unordered_map>
-
-#include "atomic.h"
-#include "buffer_pool.h"
-#include "sdk_core.h"
-#include "noncopyable.h"
-#include "user_msg.h"
-namespace dataproxy_sdk
-{
-// groupid:packqueue=1:1
-class PackQueue
-{
- private:
- char* data_;
- std::vector<UserMsgPtr> msg_set_;
- uint32_t data_capacity_; // pack_size+extend_pack_size
- uint32_t cur_len_; //data+additional attr
- AtomicInt pack_redotime_cnt_; //msg_cnt, report time is illegal
- AtomicInt pack_err_; //pack err metrics
- uint64_t first_use_;
- uint64_t last_use_;
- uint64_t data_time_; //data time
- std::string inlong_group_id_;
- std::string inlong_stream_id_;
- uint16_t groupId_num_;
- uint16_t streamId_num_;
- std::string topic_desc_; // inlong_group_id=xxx&inlong_stream_id=xxx
- uint32_t msg_type_;
- mutable std::mutex mutex_;
-
- public:
- AtomicUInt success_num_; //ack msg num in every one min
- AtomicUInt total_success_num_; //total ack msg num
-
- AtomicUInt pack_num_; //package num in every one min
-
- public:
- PackQueue(const std::string& inlong_group_id, const std::string& inlong_stream_id);
- ~PackQueue();
-
- int32_t sendMsg(const std::string& msg,
- const std::string& inlong_group_id,
- const std::string& inlong_stream_id,
- const std::string& client_ip,
- uint64_t report_time,
- UserCallBack cb);
- bool packOperate(char* pack_data, uint32_t& out_len, uint32_t uniq_id);
- void checkQueue(bool isLastPack); //if it's last pack, pack and send the rest data
- uint32_t curLen() const { return cur_len_; }
- void setCurLen(const uint32_t& cur_len) { cur_len_ = cur_len; }
- uint64_t firstUse() const { return first_use_; }
- void setFirstUse(const uint64_t& first_use) { first_use_ = first_use; }
- uint64_t lastUse() const { return last_use_; }
- void setLastUse(const uint64_t& last_use) { last_use_ = last_use; }
- uint64_t dataTime() const { return data_time_; }
- void setDataTime(const uint64_t& data_time) { data_time_ = data_time; }
- char* data() const { return data_; }
-
- std::string inlong_group_id() const { return inlong_group_id_; }
- void setGroupid(const std::string& inlong_group_id) { inlong_group_id_ = inlong_group_id; }
-
- void increasePackErr() { pack_err_.increment(); }
-
- std::string topicDesc() const { return topic_desc_; }
-
- private:
- bool isTriggerPack(uint64_t msg_time, int32_t msg_len);
- int32_t writeToBuf();
- int32_t appendMsg(const std::string& msg, std::string client_ip, int64_t report_time, UserCallBack call_back);
- bool isZipAndOperate(std::string& res, uint32_t real_cur_len);
- inline void resetPackQueue()
- {
- memset(data_, 0x0, data_capacity_);
- cur_len_ = 0;
- msg_set_.clear();
- }
-};
-
-using PackQueuePtr = std::shared_ptr<PackQueue>;
-
-// all packqueues
-class GlobalQueues : noncopyable
-{
- private:
- std::unordered_map<std::string, PackQueuePtr> queues_;
- mutable std::mutex mutex_;
- std::thread worker_; //pack thread
- bool exit_flag_;
-
- public:
- GlobalQueues() : exit_flag_(false){};
- virtual ~GlobalQueues();
- void startCheckSubroutine();
- virtual PackQueuePtr getPackQueue(const std::string& inlong_group_id, const std::string& inlong_stream_id);
- void closeCheckSubroutine() { exit_flag_ = true; }
- void printAck();
- void printTotalAck();
-
- void showState();
-
- private:
- void checkPackQueueSubroutine();
-};
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_PACK_QUEUE_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/proxylist_config.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/proxylist_config.cc
deleted file mode 100644
index 1ad00d659d..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/proxylist_config.cc
+++ /dev/null
@@ -1,989 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "proxylist_config.h"
-
-#include <algorithm>
-#include <chrono>
-#include <cstdlib>
-#include <functional>
-#include <iostream>
-#include <map>
-#include <rapidjson/document.h>
-#include <stdlib.h>
-
-#include "sdk_constant.h"
-#include "executor_thread_pool.h"
-#include "ini_help.h"
-#include "logger.h"
-#include "pack_queue.h"
-#include "socket_connection.h"
-#include "tc_api.h"
-#include "utils.h"
-namespace dataproxy_sdk
-{
- ClusterProxyList::ClusterProxyList()
- : cluster_id_(-1), size_(0), load_(0), is_inter_visit_(false), switch_value_(0), get_flag_(false), msg_num_(0), active_proxy_num_(-1), backup_proxy_num_(-1)
- {
- }
-
- ClusterProxyList::~ClusterProxyList() { clearAllConn(); }
-
- bool ClusterProxyList::isNeedLoadBalance()
- {
- // #if 0
- if (!g_config.enable_heart_beat_ || !g_config.heart_beat_interval_)
- {
- return false;
- }
- if (active_proxy_num_ < 0)
- {
- LOG_ERROR("active_proxy_num_:%d is negative", active_proxy_num_);
- }
- if (load_ <= 0 || backup_proxy_num_ == 0)
- {
- return false;
- }
- // #endif
- return true;
- }
-
- void ClusterProxyList::addBusAddress(const ProxyInfoPtr &proxy_info)
- {
- proxylist_.push_back(proxy_info);
- unused_proxylist_[proxy_info->getString()] = proxy_info;
- }
-
- void ClusterProxyList::initUnusedBus()
- {
- for (auto it : proxylist_)
- {
- unused_proxylist_.emplace(it->getString(), it);
- }
- }
-
- void ClusterProxyList::setActiveAndBackupBusNum(const int32_t &default_set)
- {
- active_proxy_num_ = std::min(default_set, size_);
- LOG_INFO("active proxy num in config is %d, real active proxy num is %d, available proxy num is %d", default_set, active_proxy_num_, size_);
- backup_proxy_num_ = std::max(0, std::min(size_ - active_proxy_num_ - 1, constants::kBackupBusNum));
- if (isNeedLoadBalance())
- {
- LOG_INFO("cluster(id:%d) need balance, backup proxy num is %d", cluster_id_, backup_proxy_num_);
- }
- else
- {
- backup_proxy_num_ = 0;
- LOG_INFO("cluster(id:%d) don't do balance, cluster load:%d, left_avaliable proxy num:%d", cluster_id_, load_, backup_proxy_num_);
- }
- }
-
- int32_t ClusterProxyList::initConn()
- {
- unique_write_lock<read_write_mutex> rdlck(rwmutex_);
- return createConnSet();
- }
-
- int32_t ClusterProxyList::createConnSet()
- {
- int err_count = 0;
- for (int i = 0; i < active_proxy_num_; i++)
- {
- auto res = createRandomConnForActiveSet();
- if (!res)
- {
- ++err_count;
-
- }
- LOG_INFO("add conn%s in active_proxy_set", res->getRemoteInfo().c_str());
- }
-
- if (isNeedLoadBalance())
- {
- for (int i = 0; i < backup_proxy_num_; i++)
- {
- auto res = createRandomConnForBackupSet();
- if (!res)
- {
- ++err_count;
- }
- LOG_INFO("add conn%s in backup_proxy_set", res->getRemoteInfo().c_str());
- }
- }
- return err_count;
- }
-
- void ClusterProxyList::clearAllConn()
- {
- unique_write_lock<read_write_mutex> rdlck(rwmutex_);
-
- for (auto &it : active_proxy_set_)
- {
- if (it.second)
- {
- it.second->connClose();
- }
- }
- active_proxy_set_.clear();
- for (auto &it : backup_proxy_set_)
- {
- if (it.second)
- {
- it.second->connClose();
- }
- }
- backup_proxy_set_.clear();
- }
-
- ConnectionPtr ClusterProxyList::getSendConn()
- {
- unique_write_lock<read_write_mutex> wtlck(rwmutex_); // FIXME: is readlock ok?
-
- if (active_proxy_set_.empty())
- {
- LOG_ERROR("cluster(id:%d) active_proxy_set is empty", cluster_id_);
- return nullptr;
- }
-
- ConnectionPtr res = nullptr;
-
- ++msg_num_;
- int32_t rr_idx = msg_num_ % active_proxy_set_.size();
- auto rr_con = std::next(std::begin(active_proxy_set_), rr_idx)->second;
-
- srand((uint32_t)Utils::getCurrentMsTime());
- int32_t rand_idx = random() % active_proxy_set_.size();
- auto rand_con = std::next(std::begin(active_proxy_set_), rand_idx)->second;
-
- if (rr_con->getWaitingSend() < rand_con->getWaitingSend()) // choosing less waiting conn
- {
- res = rr_con;
- }
- else
- {
- res = rand_con;
- }
-
- if (res->isStop())
- {
- LOG_INFO("conn %s is closed, create new conn for send", res->getRemoteInfo().c_str());
- active_proxy_set_.erase(res->getRemoteInfo());
-
- auto old_proxyinfo = res->getBusInfo();
-
- if (unused_proxylist_.empty()) //if there is no available proxy
- {
- unused_proxylist_[old_proxyinfo->getString()] = old_proxyinfo;
-
- res = createRandomConnForActiveSet();
- }
- else
- {
- res = createRandomConnForActiveSet();
-
- unused_proxylist_[old_proxyinfo->getString()] = old_proxyinfo;
- }
- }
-
- return res;
- }
-
- ConnectionPtr ClusterProxyList::createRandomConnForActiveSet() { return createRandomConn(active_proxy_set_); }
-
- ConnectionPtr ClusterProxyList::createRandomConnForBackupSet() { return createRandomConn(backup_proxy_set_); }
-
- ConnectionPtr ClusterProxyList::createRandomConn(std::unordered_map<std::string, ConnectionPtr> &conn_set)
- {
- if (!g_executors)
- return nullptr;
- if (unused_proxylist_.empty())
- {
- clearInvalidConns(); // proxy侧短时间内全部断连，先清除close的conn
- }
- if (unused_proxylist_.empty())
- {
- LOG_ERROR("all proxyes in proxylist have already established connections, something is wrong！");
- return nullptr;
- }
-
- srand((uint32_t)Utils::getCurrentMsTime());
- int32_t rand_idx = random() % unused_proxylist_.size();
- auto proxy_info = std::next(std::begin(unused_proxylist_), rand_idx)->second;
- // create conn
- auto executor = g_executors->nextExecutor();
- if(!executor){
- return nullptr;
- }
- auto res = std::make_shared<Connection>(executor, proxy_info);
- if (!res)
- {
- LOG_ERROR("failed to create new connection %s", proxy_info->getString().c_str());
- return nullptr;
- }
- LOG_DEBUG("create new connection: post %s connect request successfully on network_thread(id:%d)", proxy_info->getString().c_str(),
- executor->threadId());
- conn_set[proxy_info->getString()] = res;
- unused_proxylist_.erase(proxy_info->getString());
- return res;
- }
-
- // if switch_value_, cluster_id_ or size_ changes, return true
- bool ClusterProxyList::enableUpdate(const ClusterProxyListPtr &other)
- {
- unique_read_lock<read_write_mutex> rdlck(rwmutex_);
-
- if (this->switch_value_ != other->switchValue())
- {
- LOG_INFO("proxy ip switch_value is diff, new:%d, old:%d", other->switchValue(), this->switch_value_);
- return true;
- }
- if (this->cluster_id_ != other->clusterId())
- {
- LOG_INFO("proxy ip cluster_id is diff, new:%d, old:%d", other->clusterId(), this->cluster_id_);
- return true;
- }
- if (this->size_ != other->size())
- {
- LOG_INFO("proxy ip size is diff, new:%d, old:%d", other->size(), this->size_);
- return true;
- }
- return false;
- }
-
- void ClusterProxyList::clearInvalidConns()
- {
-
- // remove invalid conn in active set
- for (auto it = active_proxy_set_.begin(); it != active_proxy_set_.end();)
- {
- if (it->second->isStop())
- {
- LOG_INFO("active_proxy_set remove stop conn:%s", it->second->getRemoteInfo().c_str());
- unused_proxylist_.emplace(it->second->getRemoteInfo(), it->second->getBusInfo()); //close and add this proxyinfo into unused
- it = active_proxy_set_.erase(it);
- continue;
- }
- ++it;
- }
- // remove invalid conn in backup set
- for (auto it = backup_proxy_set_.begin(); it != backup_proxy_set_.end();)
- {
- if (it->second->isStop())
- {
- LOG_INFO("backup_proxy_set_ remove stop conn:%s", it->second->getRemoteInfo().c_str());
- unused_proxylist_.emplace(it->second->getRemoteInfo(), it->second->getBusInfo()); //close and add this proxyinfo into unused
- it = backup_proxy_set_.erase(it);
- continue;
- }
- ++it;
- }
- }
-
- void ClusterProxyList::keepConnsAlive()
- {
- unique_read_lock<read_write_mutex> rdlck(rwmutex_);
- for (auto it : active_proxy_set_)
- {
- it.second->sendHB(isNeedLoadBalance());
- }
- for (auto it : backup_proxy_set_)
- {
- it.second->sendHB(isNeedLoadBalance());
- }
- }
-
- void ClusterProxyList::balanceConns()
- {
- std::map<std::string, int32_t> active_load;
- std::map<std::string, int32_t> backup_load;
- unique_write_lock<read_write_mutex> wtlck(rwmutex_);
-
- if (!isNeedLoadBalance() || backup_proxy_set_.empty())
- return; //无需进行balance
-
- for (auto &it : active_proxy_set_)
- {
- int32_t avg_load = it.second->getAvgLoad();
- if (avg_load >= 0)
- {
- active_load[it.first] = avg_load;
- LOG_DEBUG("active conn:%s, avgLoad:%d", it.first.c_str(), avg_load);
- }
- }
- for (auto &it : backup_proxy_set_)
- {
- int32_t avg_load = it.second->getAvgLoad();
- if (avg_load >= 0)
- {
- backup_load[it.first] = avg_load;
- LOG_DEBUG("backup conn:%s, avgLoad:%d", it.first.c_str(), avg_load);
- }
- }
-
- // avg_load desc sort
- std::vector<PAIR> active_list(active_load.begin(), active_load.end());
- std::sort(active_list.begin(), active_list.end(), &Utils::downValueSort);
-
- // avg_load asec sort
- std::vector<PAIR> backup_list(backup_load.begin(), backup_load.end());
- std::sort(backup_list.begin(), backup_list.end(), &Utils::upValueSort);
-
- // int32_t small_size = active_list.size() < backup_list.size() ? active_list : backup_list;
- int32_t small_size = 1;
- for (int32_t i = 0; i < small_size; i++)
- {
- if (active_list.empty() || backup_list.empty())
- break;
-
- if (active_list[i].second - backup_list[i].second >= load_) // do switch
- {
- LOG_INFO("do balance, active conn:%s, load:%d <--> backup conn:%s, load:%d", active_list[i].first.c_str(), active_list[i].second,
- backup_list[i].first.c_str(), backup_list[i].second);
-
- ConnectionPtr &tmp = active_proxy_set_[active_list[i].first];
- active_proxy_set_.erase(active_list[i].first);
- active_proxy_set_[backup_list[i].first] = backup_proxy_set_[backup_list[i].first];
- backup_proxy_set_.erase(backup_list[i].first);
- backup_proxy_set_[tmp->getRemoteInfo()] = tmp;
- }
- }
- }
-
- void ClusterProxyList::updateBackupConns()
- {
- unique_write_lock<read_write_mutex> wtlck(rwmutex_);
-
- if (!isNeedLoadBalance())
- return;
-
- // update backup conns
- for (auto it : backup_proxy_set_)
- {
- if (it.second)
- {
- LOG_DEBUG("update backup_conns regularly, close old conns and then create new conns");
- it.second->connClose();
- unused_proxylist_.emplace(it.second->getRemoteInfo(), it.second->getBusInfo()); //lose and add this proxyinfo into unused
- }
- }
- backup_proxy_set_.clear();
- for (int i = 0; i < backup_proxy_num_; i++)
- {
- auto res = createRandomConnForBackupSet();
- if (!res)
- {
- LOG_ERROR("create backup conn error, check it");
- continue;
- }
- LOG_DEBUG("new create conn%s in backup_proxy_set", res->getRemoteInfo().c_str());
- }
- }
-
- GlobalCluster::GlobalCluster()
- : groupid2cluster_rwmutex_(), update_flag_(false), cond_mutex_(), cond_(), exit_flag_(false), timer_worker_(std::make_shared<ExecutorThread>(100)), clear_timer_(timer_worker_->createSteadyTimer()), doBalance_timer_(timer_worker_->createSteadyTimer()), updateBackup_timer_(timer_worker_->createSteadyTimer()), printAckNum_timer_(timer_worker_->createSteadyTimer())
- {
- clear_timer_->expires_after(std::chrono::seconds(kClearTimerSecond));
- clear_timer_->async_wait([this](const std::error_code &ec)
- { clearInvalidConn(ec); });
-
- // whether need do balance
- if (g_config.enable_heart_beat_ && g_config.heart_beat_interval_ > 0)
- {
- keepAlive_timer_ = timer_worker_->createSteadyTimer();
- keepAlive_timer_->expires_after(std::chrono::seconds(g_config.heart_beat_interval_));
- keepAlive_timer_->async_wait([this](const std::error_code &ec)
- { keepConnAlive(ec); });
- }
-
- doBalance_timer_->expires_after(std::chrono::minutes(kDoBalanceMin));
- doBalance_timer_->async_wait([this](const std::error_code &ec)
- { doBalance(ec); });
-
- printAckNum_timer_->expires_after(std::chrono::minutes(kPrintAckNumMin));
- printAckNum_timer_->async_wait([this](const std::error_code &ec)
- { printAckNum(ec); });
-
- updateBackup_timer_->expires_after(std::chrono::seconds(kUpdateBackupSecond + 3));
- updateBackup_timer_->async_wait([this](const std::error_code &ec)
- { updateBackup(ec); });
-
- }
-
- GlobalCluster::~GlobalCluster()
- {
- // FIXME:need other close work?
- timer_worker_->close();
-
- closeBuslistUpdate();
- if (worker_.joinable())
- {
- worker_.join();
- }
- }
-
- void GlobalCluster::closeBuslistUpdate()
- {
- exit_flag_ = true;
-
- std::unique_lock<std::mutex> con_lck(cond_mutex_);
- update_flag_ = true;
- con_lck.unlock();
- cond_.notify_one();
- }
-
- int32_t GlobalCluster::initBuslistAndCreateConns()
- {
- for (auto &inlong_group_id : g_config.inlong_group_ids_)
- {
- groupid2cluster_map_[inlong_group_id] = -1;
- }
-
- doUpdate();
-
- return 0;
- }
-
- void GlobalCluster::startUpdateSubroutine() { worker_ = std::thread(&GlobalCluster::updateSubroutine, this); }
-
- //FIXME: improve, using getconn err num, if it exceeds a limit, return errcode when user send next
- ConnectionPtr GlobalCluster::getSendConn(const std::string &inlong_group_id)
- {
- unique_read_lock<read_write_mutex> rdlck1(groupid2cluster_rwmutex_);
- auto it1 = groupid2cluster_map_.find(inlong_group_id);
- if (it1 == groupid2cluster_map_.end())
- {
- LOG_ERROR("there is no proxylist and connection for inlong_group_id:%s , please check inlong_group_id/url or retry later", inlong_group_id.c_str());
- return nullptr;
- }
-
- unique_read_lock<read_write_mutex> rdlck2(cluster_set_rwmutex_);
- auto it2 = cluster_set_.find(it1->second);
- if (it2 == cluster_set_.end())
- {
- LOG_ERROR("there is no cluster(id:%d) for inlong_group_id:%s in cluster_set, please check inlong_group_id/url or retry later ", it1->second, inlong_group_id.c_str());
- return nullptr;
- }
-
- return it2->second->getSendConn();
- }
-
- ConnectionPtr GlobalCluster::createActiveConn(const std::string &inlong_group_id, int32_t pool_id)
- {
- if (1 == user_exit_flag.get())// if user is closing sdk
- {
- return nullptr;
- }
-
- unique_read_lock<read_write_mutex> rdlck1(groupid2cluster_rwmutex_);
- auto it1 = groupid2cluster_map_.find(inlong_group_id); // inlong_group_id->cluster_id
- if (it1 == groupid2cluster_map_.end()) // all proxy conn are broken
- {
- LOG_ERROR("there is no proxylist or avaliable connection for inlong_group_id:%s , please check inlong_group_id/url or retry later", inlong_group_id.c_str());
- return nullptr;
- }
- unique_read_lock<read_write_mutex> rdlck2(cluster_set_rwmutex_);
- auto it2 = cluster_set_.find(it1->second); // cluster_id->proxylist
- if (it2 == cluster_set_.end())
- {
- LOG_ERROR("there is no cluster(id:%d) for inlong_group_id:%s in cluster_set, please check inlong_group_id/url or retry later", it1->second, inlong_group_id.c_str());
- return nullptr;
- }
-
- unique_write_lock<read_write_mutex> wtlck(it2->second->rwmutex_);
-
- auto res = it2->second->createRandomConnForActiveSet();
- return res;
- }
-
- void GlobalCluster::updateSubroutine()
- {
- LOG_INFO("proxylist update thread start");
-
- while (true)
- {
- std::unique_lock<std::mutex> con_lck(cond_mutex_);
- if (cond_.wait_for(con_lck, std::chrono::minutes(g_config.proxy_update_interval_), [this]()
- { return update_flag_; }))
- {
- if (exit_flag_)
- break;
- update_flag_ = false;
- con_lck.unlock();
- LOG_DEBUG("new inlong_group_id is added, update proxylist");
- doUpdate(); // FIXME:improve, only update new groupid
- }
- else
- {
- LOG_INFO("proxy update interval is %d mins, update proxylist", g_config.proxy_update_interval_);
- doUpdate();
- }
- }
- LOG_INFO("proxylist update thread exit");
- }
-
- // add inlong_group_id's proxylist into groupid2cluster_map_, if it is new, trigger updating groupid2cluster_map and cluster_set
- int32_t GlobalCluster::addBuslist(const std::string &inlong_group_id)
- {
- {
- unique_read_lock<read_write_mutex> rdlck(groupid2cluster_rwmutex_);
- auto it = groupid2cluster_map_.find(inlong_group_id);
- if (it != groupid2cluster_map_.end())
- {
- return 0;
- }
- }
- //not exist, add
- {
- unique_write_lock<read_write_mutex> wtlck(groupid2cluster_rwmutex_);
- groupid2cluster_map_.emplace(inlong_group_id, -1);
- }
-
- //set proxy update notification
- std::unique_lock<std::mutex> con_lck(cond_mutex_);
- update_flag_ = true;
- con_lck.unlock();
- cond_.notify_one();
-
- LOG_DEBUG("add inlong_group_id:%s to global groupid2cluster_map, and set notify proxy updating", inlong_group_id.c_str());
- return 0;
- }
-
- void GlobalCluster::doUpdate()
- {
- if (groupid2cluster_map_.empty())
- {
- LOG_INFO("empty inlong_group_id, no need to update proxylist");
- return;
- }
-
- std::ofstream outfile;
- outfile.open(".proxy_list.ini.tmp", std::ios::out | std::ios::trunc);
- int32_t groupId_count = 0; //flush to file, record index and count
-
- {
- unique_write_lock<read_write_mutex> wtlck(groupid2cluster_rwmutex_);
-
- // for (auto& it : proxylist_map_)
- for (auto &groupid2cluster : groupid2cluster_map_)
- {
- //拼接tdm请求的url
- std::string url;
- if (g_config.enable_proxy_URL_from_cluster_)
- url = g_config.proxy_cluster_URL_;
- else
- {
- url = g_config.proxy_URL_ + "/" + groupid2cluster.first;
- }
- std::string post_data = "ip=" + g_config.ser_ip_ + "&version=" + constants::kTDBusCAPIVersion + "&protocolType=" + constants::kProtocolType;
- LOG_WARN("get inlong_group_id:%s proxy cfg url:%s, post_data:%s", groupid2cluster.first.c_str(), url.c_str(), post_data.c_str());
-
- // request proxylist from mananer, if failed multi-times, read from local cache file
- std::string meta_data;
- int32_t ret;
- for (int i = 0; i < constants::kMaxRequestTDMTimes; i++)
- {
- HttpRequest request = {url, g_config.proxy_URL_timeout_, g_config.need_auth_, g_config.auth_id_, g_config.auth_key_, post_data};
- ret = Utils::requestUrl(meta_data, &request);
- if (!ret)
- {
- break;
- } //request success
- }
-
- if (!ret) // success
- {
- LOG_WARN("get inlong_group_id:%s proxy json list from tdm: %s", groupid2cluster.first.c_str(), meta_data.c_str());
- }
- else //request manager error
- {
- LOG_ERROR("failed to request inlong_group_id:%s proxylist from tdm, has tried max_times(%d)", groupid2cluster.first.c_str(),
- constants::kMaxRequestTDMTimes);
-
- if (groupid2cluster.second != -1 && cluster_set_.find(groupid2cluster.second) != cluster_set_.end())
- {
- LOG_WARN("failed to request inlong_group_id:%s proxylist from tdm, use previous proxylist", groupid2cluster.first.c_str());
- continue;
- }
- else //new groupid, try to read from cache proxylist
- {
- LOG_WARN("failed to request inlong_group_id:%s proxylist from tdm, also no previous proxylist, then try to find from cache file",
- groupid2cluster.first.c_str());
- auto it = cache_groupid2metaInfo_.find(groupid2cluster.first);
- if (it != cache_groupid2metaInfo_.end())
- {
- meta_data = it->second;
- LOG_WARN("get inlong_group_id:%s proxy json from cache file: %s", groupid2cluster.first.c_str(), meta_data.c_str());
- }
- else
- {
- LOG_ERROR("failed to find inlong_group_id:%s proxylist from cache file", groupid2cluster.first.c_str());
- continue;
- }
- }
- }
-
- ClusterProxyListPtr new_proxylist_cfg = std::make_shared<ClusterProxyList>();
-
- ret = parseAndGet(groupid2cluster.first, meta_data, new_proxylist_cfg);
- if (ret)
- {
- LOG_ERROR("failed to parse inlong_group_id:%s json proxylist", groupid2cluster.first.c_str());
- continue;
- }
-
- unique_write_lock<read_write_mutex> wtlck_cluster_set(cluster_set_rwmutex_);
-
- cache_groupid2metaInfo_[groupid2cluster.first] = meta_data; //for disaster
-
- // #if 0
-
- // case1. new groupid, but there is its clusterid in memory
- if (groupid2cluster.second == -1 && cluster_set_.find(new_proxylist_cfg->clusterId()) != cluster_set_.end())
- {
- auto cluster_id = new_proxylist_cfg->clusterId();
- if (cluster_set_[cluster_id]->enableUpdate(new_proxylist_cfg))
- { //已有的cluster需要更新
- new_proxylist_cfg->initConn();
- cluster_set_[cluster_id] = new_proxylist_cfg;
- LOG_INFO("update cluster(id:%d) info and connections", cluster_id);
- }
- groupid2cluster.second = cluster_id;
- LOG_INFO("add inlong_group_id:%s to groupid2cluster, its cluster(id:%d) is already in global_cluster", groupid2cluster.first.c_str(), cluster_id);
- continue;
- }
-
- // case2. new groupid, there is no its clusterid in memory, update groupid2cluster, add it into cluster
- if (groupid2cluster.second == -1 && cluster_set_.find(new_proxylist_cfg->clusterId()) == cluster_set_.end())
- {
- groupid2cluster.second = new_proxylist_cfg->clusterId();
- new_proxylist_cfg->initConn();
- cluster_set_[groupid2cluster.second] = new_proxylist_cfg;
- LOG_INFO("add inlong_group_id:%s to cluster(id:%d) map in global_cluster, and init connections completely", groupid2cluster.first.c_str(),
- new_proxylist_cfg->clusterId());
-
- continue;
- }
-
- // case3. already existing groupid, whether needs update
- if (cluster_set_[groupid2cluster.second]->enableUpdate(new_proxylist_cfg))
- {
- new_proxylist_cfg->initConn();
- cluster_set_[groupid2cluster.second] = new_proxylist_cfg;
- LOG_INFO("update cluster(id:%d) info and connections", groupid2cluster.second);
- }
- // #endif
- }
- // flush cache_groupid2metaInfo to file
- for (auto &it : cache_groupid2metaInfo_)
- {
- writeMetaData2File(outfile, groupId_count, it.first, it.second);
- groupId_count++;
- }
- }
-
- if (outfile)
- {
- if (groupId_count)
- {
- outfile << "[main]" << std::endl;
- outfile << "groupId_count=" << groupId_count << std::endl;
- }
- outfile.close();
- }
- if (groupId_count)
- rename(".proxy_list.ini.tmp", ".proxy_list.ini");
- }
-
- void GlobalCluster::writeMetaData2File(std::ofstream &file, int32_t groupId_index, const std::string &inlong_group_id, const std::string &meta_data)
- {
- file << "[inlong_group_id" << groupId_index << "]" << std::endl;
- file << "inlong_group_id=" << inlong_group_id << std::endl;
- file << "proxy_cfg=" << meta_data << std::endl;
- }
-
- int32_t GlobalCluster::readCacheBuslist()
- {
- IniFile ini = IniFile();
- if (ini.load(".proxy_list.ini"))
- {
- LOG_INFO("there is no proxylist cache file");
- return 1;
- }
- int32_t groupId_count = 0;
- if (ini.getInt("main", "groupId_count", &groupId_count))
- {
- LOG_WARN("failed to parse .proxylist.ini file");
- return 1;
- }
- for (int32_t i = 0; i < groupId_count; i++)
- {
- std::string groupidlist = "inlong_group_id" + std::to_string(i);
- std::string inlong_group_id, proxy;
- if (ini.getString(groupidlist, "inlong_group_id", &inlong_group_id))
- {
- LOG_WARN("failed to get %s name from cache file", inlong_group_id.c_str());
- continue;
- }
- if (ini.getString(groupidlist, "proxy_cfg", &proxy))
- {
- LOG_WARN("failed to get %s cache proxylist", inlong_group_id.c_str());
- continue;
- }
- LOG_INFO("read cache file, inlong_group_id:%s, proxy_cfg:%s", inlong_group_id.c_str(), proxy.c_str());
- cache_groupid2metaInfo_[inlong_group_id] = proxy;
- }
-
- return 0;
- }
-
- //parse proxylist meta
- //{"success":true,"errMsg":null,"data":{"clusterId":1,"isIntranet":null,"isSwitch":null,"load":20,"nodeList":[{"id":1,"ip":"127.0.0.1.160","port":46801}]}}
- int32_t GlobalCluster::parseAndGet(const std::string &inlong_group_id, const std::string &meta_data, ClusterProxyListPtr proxylist_config)
- {
- rapidjson::Document doc;
- if (doc.Parse(meta_data.c_str()).HasParseError())
- {
- LOG_ERROR("failed to parse meta_data, error:(%d:%d)", doc.GetParseError(), doc.GetErrorOffset());
- return SDKInvalidResult::kErrorParseJson;
- }
-
- if (!(doc.HasMember("success") && doc["success"].IsBool() && doc["success"].GetBool()))
- {
- LOG_ERROR("failed to get proxy_list of inlong_group_id:%s, success: not exist or false", inlong_group_id.c_str());
- return SDKInvalidResult::kErrorParseJson;
- }
- // check data valid
- if (!doc.HasMember("data") || doc["data"].IsNull())
- {
- LOG_ERROR("failed to get proxy_list of inlong_group_id:%s, data: not exist or null", inlong_group_id.c_str());
- return SDKInvalidResult::kErrorParseJson;
- }
-
- // check nodelist valid
- const rapidjson::Value &clusterInfo = doc["data"];
- if (!clusterInfo.HasMember("nodeList") || clusterInfo["nodeList"].IsNull())
- {
- LOG_ERROR("invalid nodeList of inlong_group_id:%s, not exist or null", inlong_group_id.c_str());
- return SDKInvalidResult::kErrorParseJson;
- }
-
- // check nodeList isn't empty
- const rapidjson::Value &nodeList = clusterInfo["nodeList"];
- if (nodeList.GetArray().Size() == 0)
- {
- LOG_ERROR("empty nodeList of inlong_group_id:%s", inlong_group_id.c_str());
- return SDKInvalidResult::kErrorParseJson;
- }
- // check clusterId
- if (!clusterInfo.HasMember("clusterId") || !clusterInfo["clusterId"].IsInt() || clusterInfo["clusterId"].GetInt() < 0)
- {
- LOG_ERROR("clusterId of inlong_group_id:%s is not found or not a integer", inlong_group_id.c_str());
- return SDKInvalidResult::kErrorParseJson;
- }
- else
- {
- const rapidjson::Value &obj = clusterInfo["clusterId"];
- proxylist_config->setClusterId(obj.GetInt());
- }
- // check isSwitch
- if (clusterInfo.HasMember("isSwitch") && clusterInfo["isSwitch"].IsInt() && !clusterInfo["isSwitch"].IsNull())
- {
- const rapidjson::Value &obj = clusterInfo["isSwitch"];
- proxylist_config->setSwitchValue(obj.GetInt());
- }
- else
- {
- LOG_WARN("switch of inlong_group_id:%s is not found or not a integer", inlong_group_id.c_str());
- proxylist_config->setSwitchValue(0);
- }
- // check load
- if (clusterInfo.HasMember("load") && clusterInfo["load"].IsInt() && !clusterInfo["load"].IsNull())
- {
- const rapidjson::Value &obj = clusterInfo["load"];
- proxylist_config->setLoad(obj.GetInt());
- }
- else
- {
- LOG_WARN("load of inlong_group_id:%s is not found or not a integer", inlong_group_id.c_str());
- proxylist_config->setLoad(0);
- }
- // check isIntranet
- if (clusterInfo.HasMember("isIntranet") && clusterInfo["isIntranet"].IsInt() && !clusterInfo["isIntranet"].IsNull())
- {
- const rapidjson::Value &obj = clusterInfo["isIntranet"];
- if (!obj.GetInt())
- proxylist_config->setIsInterVisit(false);
- }
- else
- {
- LOG_WARN("isIntranet of inlong_group_id:%s is not found or not a integer", inlong_group_id.c_str());
- proxylist_config->setIsInterVisit(true);
- }
- // proxy list
- for (auto &proxy: nodeList.GetArray())
- {
- std::string ip;
- int32_t port, id;
- if (proxy.HasMember("ip") && !proxy["ip"].IsNull())
- ip = proxy["ip"].GetString();
- else
- {
- LOG_ERROR("this ip info is null");
- continue;
- }
- if (proxy.HasMember("port") && !proxy["port"].IsNull())
- {
- if (proxy["port"].IsString())
- port = std::stoi(proxy["port"].GetString());
- else if (proxy["port"].IsInt())
- port = proxy["port"].GetInt();
- }
-
- else
- {
- LOG_ERROR("this ip info is null or negative");
- continue;
- }
- if (proxy.HasMember("id") && !proxy["id"].IsNull())
- {
- if (proxy["id"].IsString())
- id = std::stoi(proxy["id"].GetString());
- else if (proxy["id"].IsInt())
- id = proxy["id"].GetInt();
- }
- else
- {
- LOG_WARN("there is no id info of inlong_group_id");
- continue;
- }
- proxylist_config->addBusAddress(std::make_shared<ProxyInfo>(id, ip, port));
-
- }
- // set size
- proxylist_config->setSize(nodeList.GetArray().Size());
-
- // init unused_proxylist_
- proxylist_config->initUnusedBus();
-
- //set active_proxy_num and backup_proxy_num
- proxylist_config->setActiveAndBackupBusNum(g_config.max_active_proxy_num_);
-
- return 0;
- }
-
- void GlobalCluster::clearInvalidConn(const std::error_code &ec)
- {
- if (ec)
- return;
-
- {
- unique_read_lock<read_write_mutex> rdlck(cluster_set_rwmutex_);
- for (auto &it : cluster_set_)
- {
- unique_write_lock<read_write_mutex> wtlck(it.second->rwmutex_);
- it.second->clearInvalidConns();
- //补充连接
- int32_t count = it.second->activeBusNeedCreate();
- for (int32_t i = 0; i < count; i++)
- {
- it.second->createRandomConnForActiveSet();
- }
- count = it.second->backupBusNeedCreate();
- for (int32_t i = 0; i < count; i++)
- {
- it.second->createRandomConnForBackupSet();
- }
- }
- }
-
- clear_timer_->expires_after(std::chrono::seconds(kClearTimerSecond));
- clear_timer_->async_wait([this](const std::error_code &ec)
- { clearInvalidConn(ec); });
- }
-
- void GlobalCluster::keepConnAlive(const std::error_code &ec)
- {
- if (ec)
- return;
-
- {
- unique_read_lock<read_write_mutex> rdlck(cluster_set_rwmutex_);
- for (auto &it : cluster_set_)
- {
- it.second->keepConnsAlive();
- }
- }
-
- keepAlive_timer_->expires_after(std::chrono::seconds(g_config.heart_beat_interval_));
- keepAlive_timer_->async_wait([this](const std::error_code &ec)
- { keepConnAlive(ec); });
- }
-
- void GlobalCluster::doBalance(const std::error_code &ec)
- {
- if (ec)
- return;
-
- {
- unique_read_lock<read_write_mutex> rdlck(cluster_set_rwmutex_);
- for (auto &it : cluster_set_)
- {
- it.second->balanceConns();
- }
- }
-
- doBalance_timer_->expires_after(std::chrono::minutes(kDoBalanceMin));
- doBalance_timer_->async_wait([this](const std::error_code &ec)
- { doBalance(ec); });
- }
-
- void GlobalCluster::updateBackup(const std::error_code &ec)
- {
- if (ec)
- return;
-
- {
- unique_read_lock<read_write_mutex> rdlck(cluster_set_rwmutex_);
- for (auto &it : cluster_set_)
- {
- it.second->updateBackupConns();
- }
- }
-
- srand((uint32_t)Utils::getCurrentMsTime());
- int32_t rand_idx = random() % (constants::kPrimeSize);
-
- updateBackup_timer_->expires_after(std::chrono::seconds(kUpdateBackupSecond + constants::kPrime[rand_idx]));
- updateBackup_timer_->async_wait([this](const std::error_code &ec)
- { updateBackup(ec); });
- }
-
- void GlobalCluster::printAckNum(const std::error_code &ec)
- {
- if (ec)
- return;
- g_queues->printAck();
- g_queues->showState();
-
- printAckNum_timer_->expires_after(std::chrono::minutes(kPrintAckNumMin));
- printAckNum_timer_->async_wait([this](const std::error_code &ec)
- { printAckNum(ec); });
- }
-
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/proxylist_config.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/proxylist_config.h
deleted file mode 100644
index ed6dd7b5f7..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/proxylist_config.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_BUSLIST_CONFIG_H_
-#define DATAPROXY_SDK_BASE_BUSLIST_CONFIG_H_
-
-#include <asio.hpp>
-#include <chrono>
-#include <condition_variable>
-#include <fstream>
-#include <memory>
-#include <mutex>
-#include <string>
-#include <thread>
-#include <unordered_map>
-
-#include "sdk_core.h"
-#include "noncopyable.h"
-#include "read_write_mutex.h"
-namespace dataproxy_sdk
-{
- // proxyinfo: {"port":46801,"ip":"100.125.58.214","id":1}
- class ProxyInfo
- {
- private:
- int32_t proxy_id_;
- std::string ip_;
- int32_t port_num_;
-
- public:
- ProxyInfo(int32_t proxy_id, std::string ip, int32_t port_num) : proxy_id_(proxy_id), ip_(ip), port_num_(port_num) {}
- std::string getString() const { return "[" + ip_ + ":" + std::to_string(port_num_) + "]"; }
-
- int32_t proxyId() const { return proxy_id_; }
-
- std::string ip() const { return ip_; }
- void setIp(const std::string &ip) { ip_ = ip; }
-
- int32_t portNum() const { return port_num_; }
- void setPortNum(const int32_t &port_num) { port_num_ = port_num; }
- };
- //using ProxyInfoPtr = std::shared_ptr<BusInfo>;
-
- class ClusterProxyList;
- using ClusterProxyListPtr = std::shared_ptr<ClusterProxyList>;
- class ClusterProxyList
- {
- private:
- int32_t cluster_id_;
- std::vector<ProxyInfoPtr> proxylist_;
- int32_t size_;
- int32_t load_;
- bool is_inter_visit_;
- int32_t switch_value_;
- bool get_flag_;
- int32_t active_proxy_num_; //min(config->max_active_proxy_num_, available proxy num)
- int32_t backup_proxy_num_; //ess than (size_-active_proxy_num_-1)
-
- std::unordered_map<std::string, ConnectionPtr> active_proxy_set_; //key:ip+port
- std::unordered_map<std::string, ConnectionPtr> backup_proxy_set_;
-
- //should be initialized as proxylist
- std::unordered_map<std::string, ProxyInfoPtr> unused_proxylist_;
-
- uint32_t msg_num_; //cumulate sent msg count
-
- public:
- read_write_mutex rwmutex_;
-
- public:
- explicit ClusterProxyList();
- virtual ~ClusterProxyList();
-
- virtual bool isNeedLoadBalance();
- void addBusAddress(const ProxyInfoPtr &proxy_info);
- bool enableUpdate(const ClusterProxyListPtr &other);
- virtual int32_t initConn(); //init active conns and backup conns
- void clearAllConn();
- void clearInvalidConns();
- void keepConnsAlive();
- void balanceConns();
- void updateBackupConns();
- ConnectionPtr getSendConn();
-
- ConnectionPtr createRandomConnForActiveSet();
- ConnectionPtr createRandomConnForBackupSet();
-
- int32_t size() const { return size_; }
- void setSize(const int32_t &size) { size_ = size; }
- int32_t clusterId() const { return cluster_id_; }
- void setClusterId(const int32_t &cluster_id) { cluster_id_ = cluster_id; }
- int32_t switchValue() const { return switch_value_; }
- void setSwitchValue(const int32_t &switch_value) { switch_value_ = switch_value; }
- int32_t load() const { return load_; }
- void setLoad(const int32_t &load) { load_ = load; }
- bool isInterVisit() const { return is_inter_visit_; }
- void setIsInterVisit(bool is_inter_visit) { is_inter_visit_ = is_inter_visit; }
- bool getFlag() const { return get_flag_; }
- void setGetFlag(bool get_flag) { get_flag_ = get_flag; }
- int32_t activeBusNum() const { return active_proxy_num_; }
- void setActiveAndBackupBusNum(const int32_t &default_set);
- int32_t backupBusNum() const { return backup_proxy_num_; }
-
- int32_t backupBusNeedCreate() const
- {
- if (backup_proxy_set_.size() < backup_proxy_num_)
- return backup_proxy_num_ - backup_proxy_set_.size();
- return 0;
- }
- int32_t activeBusNeedCreate() const
- {
- if (active_proxy_set_.size() < active_proxy_num_)
- return active_proxy_num_ - active_proxy_set_.size();
- return 0;
- }
- void initUnusedBus();
-
- private:
- ConnectionPtr createRandomConn(std::unordered_map<std::string, ConnectionPtr> &conn_set);
- int32_t createConnSet();
-
- };
-
- class GlobalCluster : noncopyable
- {
- private:
- std::unordered_map<std::string, int32_t> groupid2cluster_map_; //<inlong_group_id,cluster_id>
- std::unordered_map<int32_t, ClusterProxyListPtr> cluster_set_; //key:cluster_id
-
- std::unordered_map<std::string, std::string> cache_groupid2metaInfo_; //<inlong_group_id,proxylist>, for disaster, read from local cache file
-
- read_write_mutex groupid2cluster_rwmutex_;
- read_write_mutex cluster_set_rwmutex_;
-
- bool update_flag_;
- std::mutex cond_mutex_;
- std::condition_variable cond_;
- bool exit_flag_; //exit proxylist update thread
- std::thread worker_;
- ExecutorThreadPtr timer_worker_; //clean invalid conn, send hb, and do balance betweeen activeconn and backupconn
-
- SteadyTimerPtr clear_timer_;
- SteadyTimerPtr keepAlive_timer_;
- SteadyTimerPtr doBalance_timer_; // do balance switch between active and backup
- SteadyTimerPtr printAckNum_timer_;
- SteadyTimerPtr updateBackup_timer_;
-
- enum
- {
- kClearTimerSecond = 10, // FIXME: whether need modification
- kDoBalanceMin = 5,
- kPrintAckNumMin = 1, // ack msg in one min cycle
- kUpdateBackupSecond = 300, // kDoBalanceMin
- };
-
- public:
- explicit GlobalCluster();
- ~GlobalCluster();
-
- int32_t initBuslistAndCreateConns();
- void startUpdateSubroutine();
- int32_t addBuslist(const std::string &inlong_group_id);
- virtual ConnectionPtr getSendConn(const std::string &inlong_group_id);
- ConnectionPtr createActiveConn(const std::string &inlong_group_id, int32_t pool_id);
- void closeBuslistUpdate();
- int32_t readCacheBuslist();
-
- private:
- void updateSubroutine();//update proxylist
- void doUpdate();
- int32_t parseAndGet(const std::string &inlong_group_id, const std::string &meta_data, ClusterProxyListPtr proxylist_config);
- void writeMetaData2File(std::ofstream &file, int32_t group, const std::string &inlong_group_id, const std::string &meta_data);
-
- void clearInvalidConn(const std::error_code &ec);
- void keepConnAlive(const std::error_code &ec); // send hb
- void doBalance(const std::error_code &ec);
- void updateBackup(const std::error_code &ec);
- virtual void printAckNum(const std::error_code &ec);
- // void cancelAllTimer();
- };
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_BUSLIST_CONFIG_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/read_write_mutex.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/read_write_mutex.h
deleted file mode 100644
index d231282292..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/read_write_mutex.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_READ_WRITE_MUTEX_H_
-#define DATAPROXY_SDK_BASE_READ_WRITE_MUTEX_H_
-
-#include <condition_variable>
-#include <mutex>
-
-namespace dataproxy_sdk
-{
-// wirte operation add lock：unique_read_lock<read_write_mutex> lock( rwmutex );
-// read operation add lock：unique_write_lock<read_write_mutex> lock(rwmutex);
-
-class read_write_mutex
-{
- public:
- read_write_mutex() = default;
- ~read_write_mutex() = default;
-
- read_write_mutex(const read_write_mutex&) = delete;
- read_write_mutex& operator=(const read_write_mutex&) = delete;
-
- read_write_mutex(const read_write_mutex&&) = delete;
- read_write_mutex& operator=(const read_write_mutex&&) = delete;
-
- void lock_read()
- {
- std::unique_lock<std::mutex> lock(m_mutex);
- m_cond_read.wait(lock, [this]() -> bool { return m_write_count == 0; });
- ++m_read_count;
- }
-
- void unlock_read()
- {
- std::unique_lock<std::mutex> lock(m_mutex);
- if (--m_read_count == 0 && m_write_count > 0) { m_cond_write.notify_one(); }
- }
-
- void lock_write()
- {
- std::unique_lock<std::mutex> lock(m_mutex);
- ++m_write_count;
- m_cond_write.wait(lock, [this]() -> bool { return m_read_count == 0 && !m_writing; });
- m_writing = true;
- }
-
- void unlock_write()
- {
- std::unique_lock<std::mutex> lock(m_mutex);
- if (--m_write_count == 0) { m_cond_read.notify_all(); }
- else
- {
- m_cond_write.notify_one();
- }
- m_writing = false;
- }
-
- private:
- volatile size_t m_read_count = 0;
- volatile size_t m_write_count = 0;
- volatile bool m_writing = false;
- mutable std::mutex m_mutex; // KEYPOINT: add mutable
- std::condition_variable m_cond_read;
- std::condition_variable m_cond_write;
-};
-
-template <typename _ReadWriteLock>
-class unique_read_lock
-{
- public:
- explicit unique_read_lock(_ReadWriteLock& rwLock) : m_ptr_rw_lock(&rwLock) { m_ptr_rw_lock->lock_read(); }
-
- ~unique_read_lock()
- {
- if (m_ptr_rw_lock) { m_ptr_rw_lock->unlock_read(); }
- }
-
- unique_read_lock() = delete;
- unique_read_lock(const unique_read_lock&) = delete;
- unique_read_lock& operator=(const unique_read_lock&) = delete;
- unique_read_lock(const unique_read_lock&&) = delete;
- unique_read_lock& operator=(const unique_read_lock&&) = delete;
-
- private:
- _ReadWriteLock* m_ptr_rw_lock = nullptr;
-};
-
-template <typename _ReadWriteLock>
-class unique_write_lock
-{
- public:
- explicit unique_write_lock(_ReadWriteLock& rwLock) : m_ptr_rw_lock(&rwLock) { m_ptr_rw_lock->lock_write(); }
-
- ~unique_write_lock()
- {
- if (m_ptr_rw_lock) { m_ptr_rw_lock->unlock_write(); }
- }
-
- unique_write_lock() = delete;
- unique_write_lock(const unique_write_lock&) = delete;
- unique_write_lock& operator=(const unique_write_lock&) = delete;
- unique_write_lock(const unique_write_lock&&) = delete;
- unique_write_lock& operator=(const unique_write_lock&&) = delete;
-
- void unlock()
- {
- if (m_ptr_rw_lock) { m_ptr_rw_lock->unlock_write(); }
- }
-
- private:
- _ReadWriteLock* m_ptr_rw_lock = nullptr;
-};
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_READ_WRITE_MUTEX_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/recv_buffer.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/recv_buffer.h
deleted file mode 100644
index 8e99776791..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/recv_buffer.h
+++ /dev/null
@@ -1,419 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_RECV_BUFFER_H_
-#define DATAPROXY_SDK_BASE_RECV_BUFFER_H_
-
-#include <arpa/inet.h>
-#include <assert.h>
-#include <stdint.h>
-#include <string.h>
-
-#include <algorithm>
-#include <condition_variable>
-#include <memory>
-#include <mutex>
-#include <string>
-
-#include "logger.h"
-
-namespace dataproxy_sdk
-{
-class RecvBuffer;
-using RecvBufferPtr = std::shared_ptr<RecvBuffer>;
-
-//store ack data from dataproxy
-class RecvBuffer
-{
- public:
- char* buffer_;
- size_t capacity_;
- size_t read_index_;
- size_t write_index_;
- size_t reserved_prepend_size_;
- bool has_mem_{true};
-
- public:
- enum {
- kCheapPrependSize = 0,
- kInitialSize = 1024 * 1024,
- };
-
- explicit RecvBuffer(size_t initial_size = kInitialSize, size_t reserved_prepend_size = kCheapPrependSize)
- : capacity_(reserved_prepend_size + initial_size)
- , read_index_(reserved_prepend_size)
- , write_index_(reserved_prepend_size)
- , reserved_prepend_size_(reserved_prepend_size)
- {
- buffer_ = new char[capacity_];
- has_mem_ = true;
- assert(length() == 0);
- assert(WritableBytes() == initial_size);
- assert(PrependableBytes() == reserved_prepend_size);
- }
-
- ~RecvBuffer()
- {
- if (has_mem_ && buffer_)
- {
- delete[] buffer_;
- }
- buffer_ = nullptr;
- capacity_ = 0;
- }
-
- std::string String()
- {
- char buf[1024];
- snprintf(buf, sizeof(buf), "buffer:%p,capacity:%ld,readindex:%ld,writeindex:%ld,prependsize:%ld,hasmem:%d", buffer_,
- capacity_, read_index_, write_index_, reserved_prepend_size_, has_mem_);
- return buf;
- }
-
- RecvBufferPtr Slice()
- {
- auto buff = std::make_shared<RecvBuffer>(*this);
- buff->has_mem_ = false;
- return buff;
- }
-
- void Swap(RecvBuffer& rhs)
- {
- std::swap(buffer_, rhs.buffer_);
- std::swap(capacity_, rhs.capacity_);
- std::swap(read_index_, rhs.read_index_);
- std::swap(write_index_, rhs.write_index_);
- std::swap(reserved_prepend_size_, rhs.reserved_prepend_size_);
- }
-
- // Skip advances the reading index of the buffer
- void Skip(size_t len)
- {
- if (len < length()) { read_index_ += len; }
- else
- {
- Reset();
- }
- }
-
- // Retrieve advances the reading index of the buffer
- // Retrieve it the same as Skip.
- void Retrieve(size_t len) { Skip(len); }
-
- // Truncate discards all but the first n unread bytes from the buffer
- // but continues to use the same allocated storage.
- // It does nothing if n is greater than the length of the buffer.
- void Truncate(size_t n)
- {
- if (n == 0)
- {
- read_index_ = reserved_prepend_size_;
- write_index_ = reserved_prepend_size_;
- }
- else if (write_index_ > read_index_ + n)
- {
- write_index_ = read_index_ + n;
- }
- }
-
- // Reset resets the buffer to be empty,
- // but it retains the underlying storage for use by future writes.
- // Reset is the same as Truncate(0).
- void Reset() { Truncate(0); }
-
- // Increase the capacity of the container to a value that's greater
- // or equal to len. If len is greater than the current capacity(),
- // new storage is allocated, otherwise the method does nothing.
- void Reserve(size_t len)
- {
- if (capacity_ >= len + reserved_prepend_size_) { return; }
-
- grow(len + reserved_prepend_size_);
- }
-
- // Make sure there is enough memory space to append more data with length len
- void EnsureWritableBytes(size_t len)
- {
- if (WritableBytes() < len) { grow(len); }
-
- assert(WritableBytes() >= len);
- }
-
- // ToText appends char '\0' to buffer to convert the underlying data to a c-style string text.
- // It will not change the length of buffer.
- void ToText()
- {
- AppendInt8('\0');
- UnwriteBytes(1);
- }
-
- // Write
- public:
- void Write(const void* /*restrict*/ d, size_t len)
- {
- EnsureWritableBytes(len);
- memcpy(WriteBegin(), d, len);
- assert(write_index_ + len <= capacity_);
- write_index_ += len;
- }
-
- void Append(const char* /*restrict*/ d, size_t len) { Write(d, len); }
-
- void Append(const void* /*restrict*/ d, size_t len) { Write(d, len); }
-
- void AppendInt32(int32_t x)
- {
- int32_t be32 = htonl(x);
- Write(&be32, sizeof be32);
- }
-
- void AppendInt16(int16_t x)
- {
- int16_t be16 = htons(x);
- Write(&be16, sizeof be16);
- }
-
- void AppendInt8(int8_t x) { Write(&x, sizeof x); }
-
- void PrependInt32(int32_t x)
- {
- int32_t be32 = htonl(x);
- Prepend(&be32, sizeof be32);
- }
-
- void PrependInt16(int16_t x)
- {
- int16_t be16 = htons(x);
- Prepend(&be16, sizeof be16);
- }
-
- void PrependInt8(int8_t x) { Prepend(&x, sizeof x); }
-
- // Insert content, specified by the parameter, into the front of reading index
- void Prepend(const void* /*restrict*/ d, size_t len)
- {
- assert(len <= PrependableBytes());
- read_index_ -= len;
- const char* p = static_cast<const char*>(d);
- memcpy(begin() + read_index_, p, len);
- }
-
- void UnwriteBytes(size_t n)
- {
- assert(n <= length());
- write_index_ -= n;
- }
-
- void WriteBytes(size_t n)
- {
- assert(n <= WritableBytes());
- write_index_ += n;
- }
-
- // Read
- public:
- // Peek int32_t/int16_t/int8_t with network endian
-
- uint32_t ReadUint32()
- {
- uint32_t result = PeekUint32();
- Skip(sizeof result);
- return result;
- }
-
- int32_t ReadInt32()
- {
- int32_t result = PeekInt32();
- Skip(sizeof result);
- return result;
- }
-
- int16_t ReadInt16()
- {
- int16_t result = PeekInt16();
- Skip(sizeof result);
- return result;
- }
-
- uint16_t ReadUint16()
- {
- uint16_t result = PeekUint16();
- Skip(sizeof result);
- return result;
- }
-
- int8_t ReadInt8()
- {
- int8_t result = PeekInt8();
- Skip(sizeof result);
- return result;
- }
-
- uint8_t ReadUint8()
- {
- uint8_t result = PeekUint8();
- Skip(sizeof result);
- return result;
- }
-
- std::string ToString() const { return std::string(data(), length()); }
-
- // ReadByte reads and returns the next byte from the buffer.
- // If no byte is available, it returns '\0'.
- char ReadByte()
- {
- assert(length() >= 1);
-
- if (length() == 0) { return '\0'; }
-
- return buffer_[read_index_++];
- }
-
- // UnreadBytes unreads the last n bytes returned
- // by the most recent read operation.
- void UnreadBytes(size_t n)
- {
- assert(n < read_index_);
- read_index_ -= n;
- }
-
- // Peek
- public:
- // Peek int64_t/int32_t/int16_t/int8_t with network endian
-
- uint32_t PeekUint32() const
- {
- // LOG_DEBUG("recv_buf->length() is%d", length());
- assert(length() >= sizeof(uint32_t));
- uint32_t be32 = 0;
- ::memcpy(&be32, data(), sizeof be32);
- return ntohl(be32);
- // LOG_DEBUG
- }
-
- int32_t PeekInt32() const
- {
- assert(length() >= sizeof(int32_t));
- int32_t be32 = 0;
- ::memcpy(&be32, data(), sizeof be32);
- return ntohl(be32);
- }
-
- int16_t PeekInt16() const
- {
- assert(length() >= sizeof(int16_t));
- int16_t be16 = 0;
- ::memcpy(&be16, data(), sizeof be16);
- return ntohs(be16);
- }
-
- uint16_t PeekUint16() const
- {
- assert(length() >= sizeof(uint16_t));
- uint16_t be16 = 0;
- ::memcpy(&be16, data(), sizeof be16);
- return ntohs(be16);
- }
-
- int8_t PeekInt8() const
- {
- assert(length() >= sizeof(int8_t));
- int8_t x = *data();
- return x;
- }
-
- uint8_t PeekUint8() const
- {
- assert(length() >= sizeof(uint8_t));
- uint8_t x = *data();
- return x;
- }
-
- public:
- // data returns a pointer of length Buffer.length() holding the unread portion of the buffer.
- // The data is valid for use only until the next buffer modification (that is,
- // only until the next call to a method like Read, Write, Reset, or Truncate).
- // The data aliases the buffer content at least until the next buffer modification,
- // so immediate changes to the slice will affect the result of future reads.
- const char* data() const { return buffer_ + read_index_; }
-
- char* WriteBegin() { return begin() + write_index_; }
-
- const char* WriteBegin() const { return begin() + write_index_; }
-
- // length returns the number of bytes of the unread portion of the buffer
- size_t length() const
- {
- assert(write_index_ >= read_index_);
- return write_index_ - read_index_;
- }
-
- // size returns the number of bytes of the unread portion of the buffer.
- // It is the same as length().
- size_t size() const { return length(); }
-
- // capacity returns the capacity of the buffer's underlying byte slice, that is, the
- // total space allocated for the buffer's data.
- size_t capacity() const { return capacity_; }
-
- size_t WritableBytes() const
- {
- assert(capacity_ >= write_index_);
- return capacity_ - write_index_;
- }
-
- size_t PrependableBytes() const { return read_index_; }
-
- public:
- char* begin() { return buffer_; }
-
- const char* begin() const { return buffer_; }
-
- void grow(size_t len)
- {
- if (!has_mem_) { return; }
- if (WritableBytes() + PrependableBytes() < len + reserved_prepend_size_)
- {
- // grow the capacity
- size_t n = (capacity_ << 1) + len;
- size_t m = length();
- char* d = new char[n];
- memcpy(d + reserved_prepend_size_, begin() + read_index_, m);
- write_index_ = m + reserved_prepend_size_;
- read_index_ = reserved_prepend_size_;
- capacity_ = n;
- delete[] buffer_;
- buffer_ = d;
- }
- else
- {
- // move readable data to the front, make space inside buffer
- assert(reserved_prepend_size_ < read_index_);
- size_t readable = length();
- memmove(begin() + reserved_prepend_size_, begin() + read_index_, length());
- read_index_ = reserved_prepend_size_;
- write_index_ = read_index_ + readable;
- assert(readable == length());
- assert(WritableBytes() >= len);
- }
- }
-};
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_RECV_BUFFER_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_constant.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_constant.h
deleted file mode 100644
index 9a0c9455d4..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_constant.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_CONSTANT_H_
-#define DATAPROXY_SDK_BASE_CONSTANT_H_
-
-#include <stdint.h>
-#include <string>
-
-namespace dataproxy_sdk
-{
- namespace constants
- {
- // load weight
- static const int32_t kWeight[30] = {1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 6, 6, 6, 6, 6, 12, 12, 12, 12, 12, 48, 96, 192, 384, 1000};
-
- static const int32_t kPrime[6] = {1, 3, 5, 7, 11, 13};
- static const int32_t kPrimeSize = 6;
-
- static const int32_t kMaxRequestTDMTimes = 4;
- static const int32_t kMaxRetryConnection = 20; //create conn failed more than 20 times, start sendbuf callback
- static const char kAttrFormat[] = "__addcol1__reptime=yyyymmddHHMMSS&__addcol2_ip=xxx.xxx.xxx.xxx"; // msg_type 7 body's attr format
- static const int32_t kAttrLen = strlen(kAttrFormat);
-
- static const char kTDBusCAPIVersion[] = "dataproxy_sdk_cpp-v2.0.0";
- static const char kLogName[] = "dataproxy_cpp.log";
- static const uint16_t kBinaryMagic = 0xEE01;
- static const uint32_t kBinPackMethod = 7; // msg>=7
- static const uint8_t kBinSnappyFlag = 1 << 5; // snappy flag
- static const int32_t kBackupBusNum = 4; // backup_proxy_num
-
- static const int32_t kThreadNums = 10;
- static const int32_t kSharedBufferNums=5;
- static const bool kEnableGroupidIsolation = false;
- static const int32_t kBufferNumPerGroupid = 5;
- static const bool kEnablePack = true;
- static const uint32_t kPackSize = 4096;
- static const uint32_t kPackTimeout = 3000;
- static const uint32_t kExtPackSize = 16384;
- static const bool kEnableZip = true;
- static const uint32_t kMinZipLen = 512;
-
- static const bool kEnableRetry = true;
- static const uint32_t kRetryInterval = 3000;
- static const uint32_t kRetryNum = 3;
- static const uint32_t kLogNum = 10;
- static const uint32_t kLogSize = 10;
- static const uint8_t kLogLevel = 2;
- static const uint8_t kLogFileType = 2;
- static const char kLogPath[] = "./sdklogs/";
- static const bool kLogEnableLimit = true;
-
- static const char kProxyURL[] = "http://127.0.0.1:8099/inlong/manager/openapi/dataproxy/getIpList";
- static const bool kEnableProxyURLFromCluster = false;
- static const char kProxyClusterURL[] =
- "http://127.0.0.1:8099/heartbeat/dataproxy_ip_v2?cluster_id=0&net_tag=normal";
- static const uint32_t kProxyUpdateInterval = 10;
- static const uint32_t kProxyURLTimeout = 2;
- static const uint32_t kMaxActiveProxyNum = 3;
-
- static const char kSerIP[] = "127.0.0.1";
- static const uint32_t kMaxBufPool = 50 * 1024 * 1024;
- static const uint32_t kMsgType = 7;
-
- static const bool kEnableTCPNagle = true;
- static const bool kEnableHeartBeat = true;
- static const uint32_t kHeartBeatInterval = 60;
- static const bool kEnableSetAffinity = false;
- static const uint32_t kMaskCPUAffinity = 0xff;
- static const bool kIsFromDC = false;
- static const uint16_t kExtendField = 0;
- static const char kNetTag[] = "all";
-
- static const bool kNeedAuth = false;
-
- // http basic auth
- static const char kBasicAuthHeader[] = "Authorization:";
- static const char kBasicAuthPrefix[] = "Basic";
- static const char kBasicAuthSeparator[] = " ";
- static const char kBasicAuthJoiner[] = ":";
- static const char kProtocolType [] = "TCP";
-
- static const uint32_t kMaxAttrLen = 2048;
-
- } // namespace constants
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_CONSTANT_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_core.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_core.cc
deleted file mode 100644
index a29ebbdd42..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_core.cc
+++ /dev/null
@@ -1,332 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "tc_api.h"
-
-#include <chrono>
-#include <signal.h>
-#include <stdio.h>
-#include <string>
-#include <unistd.h>
-#include <unordered_map>
-#include <vector>
-#include <algorithm>
-
-#include "atomic.h"
-#include "buffer_pool.h"
-#include "proxylist_config.h"
-#include "sdk_constant.h"
-#include "client_config.h"
-#include "logger.h"
-#include "pack_queue.h"
-#include "utils.h"
-#include "rapidjson/document.h"
-
-namespace dataproxy_sdk
-{
- AtomicInt init_flag{0};
- AtomicInt user_exit_flag{0};
- AtomicUInt g_send_msgid{0}; // msg uuid
- AtomicInt g_buf_full{0}; // used for buf full log limit
- ClientConfig g_config;
- GlobalCluster *g_clusters = nullptr;
- GlobalQueues *g_queues = nullptr;
- TotalPools *g_pools = nullptr;
- ExecutorThreadPool *g_executors = nullptr;
-
- int32_t init_helper ()
- {
- debug_init_log();
-
- LOG_WARN("dataproxy_sdk_cpp start init, version:%s", constants::kTDBusCAPIVersion);
-
- g_config.showClientConfig();
-
- // get local ip
- if (!Utils::getFirstIpAddr(g_config.ser_ip_))
- {
- LOG_WARN("not found the localHost in local OS, use user's ser_ip(%s) in config", g_config.ser_ip_.c_str());
- }
-
- if (g_config.enable_setaffinity_)
- {
- Utils::bindCPU(g_config.mask_cpu_affinity_);
- }
-
- signal(SIGPIPE, SIG_IGN);
-
- g_pools = new TotalPools();
- if (!g_pools)
- {
- LOG_ERROR("fail to init global buffer pools");
- return SDKInvalidResult::kErrorInit;
- }
- LOG_INFO("buf pools init complete");
-
- g_clusters = new GlobalCluster();
- if (!g_clusters)
- {
- LOG_ERROR("fail to init global clusterlist");
- return SDKInvalidResult::kErrorInit;
- }
- LOG_INFO("global clusterlist init complete");
-
- g_queues = new GlobalQueues();
- if (!g_queues)
- {
- LOG_ERROR("fail to init global packqueue");
- return SDKInvalidResult::kErrorInit;
- }
- LOG_INFO("global packqueue init complete");
-
- // init network threadpools
- g_executors = new ExecutorThreadPool();
- if (!g_executors)
- {
- LOG_ERROR("fail to init network threads");
- return SDKInvalidResult::kErrorInit;
- }
-
- LOG_INFO("read cache proxylist for disaster tolerance");
- g_clusters->readCacheBuslist();
-
- if (!g_config.inlong_group_ids_.empty())
- {
- int32_t ret = g_clusters->initBuslistAndCreateConns();
- // FIXME: improve, return ret to user?
- }
-
- // packqueue flush thread
- g_queues->startCheckSubroutine();
-
- // proxylist update thread
- g_clusters->startUpdateSubroutine();
-
- LOG_WARN("dataproxy_sdk_cpp init complete!");
-
- return 0;
-
- }
-
- int32_t tc_api_init(const char *config_file)
- {
-
- // one process is only initialized once
- if (!init_flag.compareAndSwap(0, 1))
- {
- LOG_ERROR("dataproxy_sdk_cpp has been initialized before!");
- return SDKInvalidResult::kMultiInit;
- }
- user_exit_flag.getAndSet(0);
-
- if (!g_config.parseConfig(config_file)){
- LOG_ERROR("dataproxy_sdk_cpp init error, something configs use default value");
- }
-
- return init_helper();
-
- }
-
-
- int32_t tc_api_init(ClientConfig& client_config)
- {
- if (!init_flag.compareAndSwap(0, 1))
- {
- return SDKInvalidResult::kMultiInit;
- }
-
- // check and proxy url
- if (client_config.proxy_URL_.empty())
- {
- init_flag.compareAndSwap(1, 0);
- return SDKInvalidResult::kErrorInit;
-
- }
- // check auth setting
- if (client_config.need_auth_ && (client_config.auth_id_.empty() || client_config.auth_key_.empty()))
- {
- init_flag.compareAndSwap(1, 0);
- return SDKInvalidResult::kErrorAuthInfo;
- }
-
- g_config = client_config;
- g_config.updateBufSize();
-
- return init_helper();
-
- }
-
- int32_t tc_api_init_ext(const char *config_file, int32_t use_def)
- {
- if (!init_flag.compareAndSwap(0, 1))
- {
- LOG_ERROR("dataproxy_sdk_cpp has been initialized before!");
- return SDKInvalidResult::kMultiInit;
- }
- user_exit_flag.getAndSet(0);
- if (!g_config.parseConfig(config_file)){
- LOG_ERROR("dataproxy_sdk_cpp init error");
-
- // don't use default, return directly
- if (!use_def)
- {
- LOG_ERROR("not using default config, dataproxy_sdk_cpp should be inited again!");
- init_flag.compareAndSwap(1, 0);
- return SDKInvalidResult::kErrorInit;
- }
-
- }
- return init_helper();
- }
-
- int32_t sendBaseMsg(const std::string msg,
- const std::string inlong_group_id,
- const std::string inlong_stream_id,
- const std::string client_ip,
- int64_t report_time,
- UserCallBack call_back)
- {
- // should init first, and close before
- if (init_flag.get() == 0 || user_exit_flag.get() == 1)
- {
- LOG_ERROR("capi has been closed, init first and then send");
- return SDKInvalidResult::kSendAfterClose;
- }
-
- // input check
- if (msg.empty() || inlong_group_id.empty() || inlong_stream_id.empty())
- {
- LOG_ERROR("invalid input, inlong_group_id:%s, inlong_stream_id:%s, msg:%s", inlong_group_id.c_str(), inlong_stream_id.c_str(), msg.c_str());
- return SDKInvalidResult::kInvalidInput;
- }
-
- // msg len check
- if (msg.size() > g_config.ext_pack_size_)
- {
- LOG_ERROR("msg len is too long, cur msg_len:%d, ext_pack_size:%d", msg.size(), g_config.ext_pack_size_);
- return SDKInvalidResult::kMsgTooLong;
- }
-
- if(g_config.enable_groupId_isolation_){
- if(std::find(g_config.inlong_group_ids_.begin(),g_config.inlong_group_ids_.end(),inlong_group_id)==g_config.inlong_group_ids_.end()){
- LOG_ERROR("inlong_group_id:%s is not specified in config file, check it", inlong_group_id.c_str());
- return SDKInvalidResult::kInvalidGroupId;
- }
- }
-
- g_clusters->addBuslist(inlong_group_id);
-
- if (!g_pools->isPoolAvailable(inlong_group_id))
- {
- if (g_buf_full.get() == 0)
- LOG_ERROR("buf pool is full, send later, datalen:%d", msg.size());
- g_buf_full.increment();
- if (g_buf_full.get() > 100)
- {
- g_buf_full.getAndSet(1);
- LOG_ERROR("buf pool is full 100 times, send later, datalen:%d", msg.size());
- }
-
- return SDKInvalidResult::kBufferPoolFull;
- }
-
- //get packqueue
- auto pack_queue = g_queues->getPackQueue(inlong_group_id, inlong_stream_id);
- if (!pack_queue)
- {
- LOG_ERROR("fail to get pack queue, inlong_group_id:%s, inlong_stream_id:%s", inlong_group_id.c_str(), inlong_stream_id.c_str());
- return SDKInvalidResult::kFailGetPackQueue;
- }
-
- return pack_queue->sendMsg(msg, inlong_group_id, inlong_stream_id, client_ip, report_time, call_back);
- }
-
- int32_t tc_api_send(const char *proxyiness_id, const char *inlong_stream_id, const char *msg, int32_t msg_len, UserCallBack call_back)
- {
- int64_t msg_time = Utils::getCurrentMsTime();
- return sendBaseMsg(msg, proxyiness_id, inlong_stream_id, "", msg_time, call_back);
- }
-
- int32_t tc_api_send_batch(const char *proxyiness_id, const char *inlong_stream_id, const char **msg_list, int32_t msg_cnt, UserCallBack call_back)
- {
- if (!msg_cnt)
- return SDKInvalidResult::kInvalidInput;
- int64_t msg_time = Utils::getCurrentMsTime();
- int32_t ret = 0;
- for (size_t i = 0; i < msg_cnt; i++)
- {
- ret = sendBaseMsg(msg_list[i], proxyiness_id, inlong_stream_id, "", msg_time, call_back);
- if (ret)
- return ret;
- }
- return 0;
- }
-
- int32_t tc_api_send_ext(const char *proxyiness_id, const char *inlong_stream_id, const char *msg, int32_t msg_len, const int64_t msg_time, UserCallBack call_back)
- {
- return sendBaseMsg(msg, proxyiness_id, inlong_stream_id, "", msg_time, call_back);
- }
-
- int32_t tc_api_send_base(const char *proxyiness_id,
- const char *inlong_stream_id,
- const char *msg,
- int32_t msg_len,
- const int64_t report_time,
- const char *client_ip,
- UserCallBack call_back)
- {
- return sendBaseMsg(msg, proxyiness_id, inlong_stream_id, client_ip, report_time, call_back);
- }
-
- int32_t tc_api_close(int32_t max_waitms)
- {
- // only exit once
- if (!init_flag.compareAndSwap(1, 0))
- {
- LOG_ERROR("dataproxy_sdk_cpp has been closed!");
- return SDKInvalidResult::kMultiExits;
- }
-
- user_exit_flag.getAndSet(1);
-
- std::this_thread::sleep_for(std::chrono::milliseconds(max_waitms));
-
- if(g_queues)
- {
- g_queues->printTotalAck(); // pring ack msg count of each groupid+streamid
- }
-
- delete g_queues;
- g_queues=nullptr;
- std::this_thread::sleep_for(std::chrono::seconds(1));
- delete g_executors;
- g_executors=nullptr;
- delete g_pools;
- g_pools=nullptr;
- delete g_clusters;
- g_clusters=nullptr;
-
- // std::this_thread::sleep_for(std::chrono::seconds(5));
-
- LOG_WARN("close dataproxy_sdk_cpp!");
-
- return 0;
- }
-
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_core.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_core.h
deleted file mode 100644
index dccf2ce1d1..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/sdk_core.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_CORE_H_
-#define DATAPROXY_SDK_BASE_CORE_H_
-
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <asio.hpp>
-
-#include "atomic.h"
-#include "user_msg.h"
-#include "client_config.h"
-
-// #include "buffer_pool.h"
-namespace dataproxy_sdk {
-class GlobalCluster;
-class GlobalQueues;
-class TotalPools;
-
-class ExecutorThread;
-using ExecutorThreadPtr = std::shared_ptr<ExecutorThread>;
-
-class ExecutorThreadPool;
-
-class Connection;
-using ConnectionPtr = std::shared_ptr<Connection>;
-
-class ProxyInfo;
-using ProxyInfoPtr = std::shared_ptr<ProxyInfo>;
-
-class SendBuffer;
-
-using TcpSocketPtr = std::shared_ptr<asio::ip::tcp::socket>;
-using SteadyTimerPtr = std::shared_ptr<asio::steady_timer>;
-using io_context_work = asio::executor_work_guard<asio::io_context::executor_type>;
-
-extern AtomicUInt g_send_msgid;
-extern ClientConfig g_config;
-extern GlobalCluster* g_clusters;
-extern GlobalQueues* g_queues;
-extern TotalPools* g_pools;
-extern ExecutorThreadPool* g_executors;
-extern AtomicInt user_exit_flag;
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_CORE_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/singleton.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/singleton.h
deleted file mode 100644
index 6e927de87a..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/singleton.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_SINGLETON_H_
-#define DATAPROXY_SDK_BASE_SINGLETON_H_
-
-#include <assert.h>
-#include <mutex>
-#include <thread>
-
-#include "noncopyable.h"
-
-namespace dataproxy_sdk {
-template <typename T>
-class Singleton : noncopyable {
-private:
- static std::once_flag once_;
- static T* value_;
-
-public:
- Singleton() = delete;
- ~Singleton() = delete;
-
- static T& instance()
- {
- std::call_once(once_, Singleton::init);
- assert(value_ != nullptr);
-
- return *value_;
- }
-
-private:
- static void init()
- {
- value_ = new T();
- }
-};
-
-template <typename T>
-std::once_flag Singleton<T>::once_;
-
-template <typename T>
-T* Singleton<T>::value_ = nullptr;
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_SINGLETON_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/user_msg.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/user_msg.h
deleted file mode 100644
index 49dbea42e6..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/user_msg.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_USER_MSG_H_
-#define DATAPROXY_SDK_BASE_USER_MSG_H_
-
-#include <memory>
-#include <stdint.h>
-#include <string>
-namespace dataproxy_sdk
-{
-// UserCallBack signature: inlong_group_id,inlong_stream_id,msg,msg_len, report_time, client_ip
-using UserCallBack = std::function<int32_t(const char*, const char*, const char*, int32_t, const int64_t, const char*)>;
-
-struct UserMsg
-{
- std::string msg;
- std::string client_ip;
- int64_t report_time;
- UserCallBack cb;
-
- int64_t user_report_time;
- std::string user_client_ip;
-
- std::string data_pack_format_attr; //"__addcol1__reptime=" + Utils::getFormatTime(data_time_) + "&__addcol2__ip=" + client_ip
- UserMsg(const std::string& mmsg,
- const std::string& mclient_ip,
- int64_t mreport_time,
- UserCallBack mcb,
- const std::string& attr,
- const std::string& u_ip,
- int64_t u_time)
- : msg(mmsg)
- , client_ip(mclient_ip)
- , report_time(mreport_time)
- , cb(mcb)
- , data_pack_format_attr(attr)
- , user_client_ip(u_ip)
- , user_report_time(u_time)
- {
- }
-};
-using UserMsgPtr = std::shared_ptr<UserMsg>;
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_USER_MSG_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/utils.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/utils.cc
deleted file mode 100644
index 33473c413e..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/utils.cc
+++ /dev/null
@@ -1,440 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "utils.h"
-
-#include <arpa/inet.h>
-#include <ctime>
-#include <errno.h>
-#include <fstream>
-#include <iostream>
-#include <iterator>
-#include <net/if.h>
-#include <netinet/in.h>
-#include <pthread.h>
-#include <regex>
-#include <sstream>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/sysinfo.h>
-#include <sys/time.h>
-
-#include "logger.h"
-#include "tc_api.h"
-#include "curl/curl.h"
-namespace dataproxy_sdk
-{
-uint16_t Utils::sequence = 0;
-uint64_t Utils::last_msstamp = 0;
-char Utils::snowflake_id[35] = {0};
-char base64_table[] = {
- 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
- 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
- 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
- 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
- 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
- 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
- 'w', 'x', 'y', 'z', '0', '1', '2', '3',
- '4', '5', '6', '7', '8', '9', '+', '/'
- };
-
-void Utils::taskWaitTime(int32_t sec)
-{
- struct timeval tv;
- tv.tv_sec = sec;
- tv.tv_usec = 0;
- int err;
- do
- {
- err = select(0, NULL, NULL, NULL, &tv);
- } while (err < 0 && errno == EINTR);
-}
-
-uint64_t Utils::getCurrentMsTime()
-{
- uint64_t ms_time = 0;
- struct timeval tv;
- gettimeofday(&tv, NULL);
- ms_time = ((uint64_t)tv.tv_sec * 1000 + tv.tv_usec / 1000);
- return ms_time;
-}
-uint64_t Utils::getCurrentWsTime()
-{
- uint64_t ws_time = 0;
- struct timeval tv;
- gettimeofday(&tv, NULL);
- ws_time = ((uint64_t)tv.tv_sec * 1000000 + tv.tv_usec);
- return ws_time;
-}
-
-std::string Utils::getFormatTime(uint64_t data_time)
-{
- struct tm timeinfo;
- char buffer[80];
-
- // time(&rawtime);
- time_t m_time = data_time / 1000;
- localtime_r(&m_time, &timeinfo);
-
- strftime(buffer, sizeof(buffer), "%Y%m%d%H%M%S", &timeinfo);
-
- return std::string(buffer);
-}
-
-size_t Utils::zipData(const char* input, uint32_t input_len, std::string& zip_res)
-{
- size_t len_after_zip = snappy::Compress((char*)input, input_len, &zip_res);
- LOG_TRACE("data zip: input len is %u, output len is %u.", input_len, len_after_zip);
- return len_after_zip;
-}
-
-char* Utils::getSnowflakeId()
-{
- std::string local_host;
- getFirstIpAddr(local_host);
- uint64_t ipaddr = htonl(inet_addr(local_host.c_str()));
- uint32_t pidid = static_cast<uint16_t>((getpid() & 0xFFFF));
- uint32_t selfid = static_cast<uint16_t>((pthread_self() & 0xFFFF00) >> 8);
-
- // 22bit ms
- uint64_t sequence_mask = -1LL ^ (-1LL << 22);
-
- uint64_t time_id = 0LL;
- uint64_t local_id = (ipaddr << 32) | (pidid << 16) | (selfid);
-
- uint64_t since_date = 1288834974657LL; // Thu, 04 Nov 2010 01:42:54 GMT
-
- // 41bit ms
- uint64_t msstamp = getCurrentMsTime();
-
- uint64_t rand = 0;
- uint64_t rand_mask = -1LL ^ (-1LL << (32 + 5 + 12));
-
- // error timestap
- if (msstamp < last_msstamp)
- {
- LOG_ERROR("ms(%llx) time less last(%llx).", msstamp, last_msstamp);
-
- //last ms
- last_msstamp = msstamp;
-
- srand(static_cast<uint32_t>(msstamp));
- rand = random();
-
- // generate id
- time_id = ((msstamp - since_date) << 22 | (rand & rand_mask));
-
- snprintf(&snowflake_id[0], sizeof(snowflake_id), "0x%.16llx%.16llx", local_id, time_id);
- return &snowflake_id[0];
- }
-
- // increase id
- if (last_msstamp == msstamp)
- {
- sequence = (sequence + 1) & sequence_mask;
-
- if (0 == sequence) { msstamp = waitNextMills(last_msstamp); }
- }
- else
- {
- sequence = 0;
- }
-
- last_msstamp = msstamp;
-
- time_id = (((msstamp - since_date) << 22) | sequence);
-
- LOG_TRACE("ms:0x%llx, ip:0x%.16llx, seq:0x:%x, selfid:%u, "
- "local_id:0x%.16llx, time_id:0x%.16llx.",
- (msstamp - since_date) << (22), ipaddr, sequence, static_cast<uint32_t>(pthread_self()), local_id, time_id);
-
- snprintf(&snowflake_id[0], sizeof(snowflake_id), "0x%.16llx%.16llx", local_id, time_id);
- return &snowflake_id[0];
-}
-
-int64_t Utils::waitNextMills(int64_t last_ms)
-{
- int64_t msstamp = getCurrentMsTime();
-
- while (msstamp <= last_ms)
- {
- msstamp = getCurrentMsTime();
- }
-
- return msstamp;
-}
-
-bool Utils::getFirstIpAddr(std::string& local_host)
-{
- int32_t sockfd;
- int32_t ip_num = 0;
- char buf[1024] = {0};
- struct ifreq* ifreq;
- struct ifreq if_flag;
- struct ifconf ifconf;
-
- ifconf.ifc_len = sizeof(buf);
- ifconf.ifc_buf = buf;
- if ((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
- {
- LOG_ERROR("open the local socket(AF_INET, SOCK_DGRAM) failure!");
- return false;
- }
-
- ioctl(sockfd, SIOCGIFCONF, &ifconf);
-
- ifreq = (struct ifreq*)buf;
- ip_num = ifconf.ifc_len / sizeof(struct ifreq);
- for (int32_t i = 0; i < ip_num; i++, ifreq++)
- {
- //exclude ipv6 addr
- if (ifreq->ifr_flags != AF_INET) { continue; }
-
- if (0 == strncmp(&ifreq->ifr_name[0], "lo", sizeof("lo"))) { continue; }
-
- memcpy(&if_flag.ifr_name[0], &ifreq->ifr_name[0], sizeof(ifreq->ifr_name));
-
- if ((ioctl(sockfd, SIOCGIFFLAGS, (char*)&if_flag)) < 0) { continue; }
-
- if ((if_flag.ifr_flags & IFF_LOOPBACK) || !(if_flag.ifr_flags & IFF_UP)) { continue; }
-
- if (!strncmp(inet_ntoa(((struct sockaddr_in*)&(ifreq->ifr_addr))->sin_addr), "127.0.0.1", 7)) { continue; }
-
- local_host = inet_ntoa(((struct sockaddr_in*)&(ifreq->ifr_addr))->sin_addr);
- close(sockfd);
- return true;
- }
- close(sockfd);
- // local_host = "127.0.0.1";
- return false;
-}
-
-bool Utils::bindCPU(int32_t cpu_id)
-{
- int32_t cpunum = get_nprocs();
- int32_t cpucore = cpu_id;
- cpu_set_t mask;
-
- if (abs(cpu_id) > cpunum)
- {
- LOG_ERROR("mask<%d> more than total cpu num<%d>.", cpu_id, cpunum);
- return false;
- }
-
- if (cpu_id < 0) { cpucore = cpunum + cpu_id; }
-
- CPU_ZERO(&mask);
- CPU_SET(cpucore, &mask);
-
- if (sched_setaffinity(0, sizeof(mask), &mask) < 0) { LOG_ERROR("set CPU affinity<%d>/<%d> errno<%d>", cpu_id, cpunum, errno); }
-
- return true;
-}
-
-std::string Utils::base64_encode(const std::string& data)
-{
- size_t in_len = data.size();
- size_t out_len = 4 * ((in_len + 2) / 3);
- std::string ret(out_len, '\0');
- size_t i;
- char *p = const_cast<char*>(ret.c_str());
-
- for (i = 0; i < in_len - 2; i += 3) {
- *p++ = base64_table[(data[i] >> 2) & 0x3F];
- *p++ = base64_table[((data[i] & 0x3) << 4) | ((int) (data[i + 1] & 0xF0) >> 4)];
- *p++ = base64_table[((data[i + 1] & 0xF) << 2) | ((int) (data[i + 2] & 0xC0) >> 6)];
- *p++ = base64_table[data[i + 2] & 0x3F];
- }
- if (i < in_len) {
- *p++ = base64_table[(data[i] >> 2) & 0x3F];
- if (i == (in_len - 1)) {
- *p++ = base64_table[((data[i] & 0x3) << 4)];
- *p++ = '=';
- }
- else {
- *p++ = base64_table[((data[i] & 0x3) << 4) | ((int) (data[i + 1] & 0xF0) >> 4)];
- *p++ = base64_table[((data[i + 1] & 0xF) << 2)];
- }
- *p++ = '=';
- }
- return ret;
-}
-
-std::string Utils::genBasicAuthCredential(const std::string& id, const std::string& key)
-{
- std::string credential = id + constants::kBasicAuthJoiner + key;
- std::string result = constants::kBasicAuthPrefix;
- result.append(constants::kBasicAuthSeparator);
- result.append(base64_encode(credential));
- return result;
-}
-
-int32_t Utils::requestUrl(std::string& res, const HttpRequest* request)
-{
- CURL* curl = NULL;
- struct curl_slist* list = NULL;
-
- curl_global_init(CURL_GLOBAL_ALL);
-
- curl = curl_easy_init();
- if (!curl)
- {
- LOG_ERROR("failed to init curl object");
- return SDKInvalidResult::kErrorCURL;
- }
-
- // http header
- list = curl_slist_append(list, "Content-Type: application/x-www-form-urlencoded");
- if ( request->need_auth && !request->auth_id.empty() && !request->auth_key.empty())
- {
- // Authorization: Basic xxxxxxxx
- std::string auth = constants::kBasicAuthHeader;
- auth.append(constants::kBasicAuthSeparator);
- auth.append(genBasicAuthCredential(request->auth_id, request->auth_key));
- LOG_INFO("request manager, auth-header:%s", auth.c_str());
- list = curl_slist_append(list, auth.c_str());
- }
- curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
-
- //set url
- curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
- curl_easy_setopt(curl, CURLOPT_URL, request->url.c_str());
- curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request->post_data.c_str());
- curl_easy_setopt(curl, CURLOPT_TIMEOUT, request->timeout);
-
- //register callback and get res
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &Utils::getUrlResponse);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, &res);
- curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
-
- //execute curl request
- CURLcode ret = curl_easy_perform(curl);
- if (ret != 0)
- {
- LOG_ERROR("%s", curl_easy_strerror(ret));
- LOG_ERROR("failed to request data from %s", request->url.c_str());
- if (curl) curl_easy_cleanup(curl);
- curl_global_cleanup();
-
- return SDKInvalidResult::kErrorCURL;
- }
-
- int32_t code;
- curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
- if (code != 200)
- {
- LOG_ERROR("tdm responsed with code %d", code);
- if (curl) curl_easy_cleanup(curl);
- curl_global_cleanup();
-
- return SDKInvalidResult::kErrorCURL;
- }
-
- if (res.empty())
- {
- LOG_ERROR("tdm return empty data");
- if (curl) curl_easy_cleanup(curl);
- curl_global_cleanup();
-
- return SDKInvalidResult::kErrorCURL;
- }
-
- //clean work
- curl_easy_cleanup(curl);
- curl_global_cleanup();
-
- return 0;
-}
-
-size_t Utils::getUrlResponse(void* buffer, size_t size, size_t count, void* response)
-{
- std::string* str = (std::string*)response;
- (*str).append((char*)buffer, size * count);
-
- return size * count;
-}
-
-bool Utils::readFile(const std::string& file_path, std::string& content)
-{
- std::ifstream f(file_path.c_str());
- if (f.fail())
- {
- LOG_ERROR("fail to read file:%s, please check file path", file_path.c_str());
- return false;
- }
- std::stringstream ss;
- ss << f.rdbuf();
- content = ss.str();
- return true;
-}
-
-static const char kWhitespaceCharSet[] = " \n\r\t\f\v";
-
-std::string Utils::trim(const std::string& source)
-{
- std::string target = source;
- if (!target.empty())
- {
- size_t foud_pos = target.find_first_not_of(kWhitespaceCharSet);
- if (foud_pos != std::string::npos) { target = target.substr(foud_pos); }
- foud_pos = target.find_last_not_of(kWhitespaceCharSet);
- if (foud_pos != std::string::npos) { target = target.substr(0, foud_pos + 1); }
- }
- return target;
-}
-
-int32_t Utils::splitOperate(const std::string& source, std::vector<std::string>& result, const std::string& delimiter)
-{
- std::string item_str;
- std::string::size_type pos1 = 0;
- std::string::size_type pos2 = 0;
- result.clear();
- if (!source.empty())
- {
- pos1 = 0;
- pos2 = source.find(delimiter);
- while (std::string::npos != pos2)
- {
- item_str = trim(source.substr(pos1, pos2 - pos1));
- pos1 = pos2 + delimiter.size();
- pos2 = source.find(delimiter, pos1);
- if (!item_str.empty()) { result.push_back(item_str); }
- }
- if (pos1 != source.length())
- {
- item_str = trim(source.substr(pos1));
- if (!item_str.empty()) { result.push_back(item_str); }
- }
- }
- return result.size();
-}
-
-std::string Utils::getVectorStr(std::vector<std::string>& vs)
-{
- std::string res;
- for (auto& it : vs)
- {
- res += it + ", ";
- }
- return res;
-}
-
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/utils.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/utils.h
deleted file mode 100644
index 708b8b34d5..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/base/utils.h
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_BASE_UTILS_H_
-#define DATAPROXY_SDK_BASE_UTILS_H_
-
-#include <stdint.h>
-#include <string>
-#include <sys/select.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <utility>
-#include <vector>
-
-#include "snappy.h"
-namespace dataproxy_sdk
-{
-using PAIR = std::pair<std::string, int32_t>;
-struct HttpRequest
-{
- std::string url;
- uint32_t timeout;
- bool need_auth;
- std::string auth_id;
- std::string auth_key;
- std::string post_data;
-
-};
-
-class Utils
-{
- private:
- static char snowflake_id[35];
- static uint16_t sequence;
- static uint64_t last_msstamp;
-
- public:
- static void taskWaitTime(int32_t sec);
- static uint64_t getCurrentMsTime();
- static uint64_t getCurrentWsTime();
- static std::string getFormatTime(uint64_t date_time); //format time: yyyymmddHHMMSS
- static size_t zipData(const char* input, uint32_t input_len, std::string& zip_res); //snappy data
- static char* getSnowflakeId(); //get 64bit snowflakeId
- static bool getFirstIpAddr(std::string& local_host);
- inline static bool isLegalTime(uint64_t report_time)
- {
- return ((report_time > 1435101567000LL) && (report_time < 4103101567000LL));
- }
- static bool bindCPU(int32_t cpu_id);
- static std::string base64_encode(const std::string& data);
- static std::string genBasicAuthCredential(const std::string& id, const std::string& key);
- static int32_t requestUrl(std::string& res, const HttpRequest* request);
- static bool readFile(const std::string& file_path, std::string& content); //read file content, save as res, return true is success
- static int32_t splitOperate(const std::string& source, std::vector<std::string>& result, const std::string& delimiter);
- static std::string getVectorStr(std::vector<std::string>& vs);
-
- static bool upValueSort(const PAIR& lhs, const PAIR& rhs) { return lhs.second < rhs.second; }
- static bool downValueSort(const PAIR& lhs, const PAIR& rhs) { return lhs.second > rhs.second; }
-
- private:
- static size_t getUrlResponse(void* buffer, size_t size, size_t count, void* response);
- static int64_t waitNextMills(int64_t last_ms);
- static std::string trim(const std::string& source);
-};
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_BASE_UTILS_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
index b5b613717b..d08d262067 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
@@ -1,21 +1,21 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
#include "sdk_conf.h"
#include <rapidjson/document.h>
@@ -97,7 +97,7 @@ void SdkConfig::defaultInit() {
log_path_ = constants::kLogPath;
// manager parameters
- manager_url_ = constants::kBusURL;
+ manager_url_ = constants::kManagerURL;
enable_manager_url_from_cluster_ = constants::kEnableBusURLFromCluster;
manager_cluster_url_ = constants::kBusClusterURL;
manager_update_interval_ = constants::kBusUpdateInterval;
@@ -243,12 +243,12 @@ void SdkConfig::InitLogParam(const rapidjson::Value &doc) {
}
}
void SdkConfig::InitBusParam(const rapidjson::Value &doc) {
- // bus_URL
+ // manager url
if (doc.HasMember("manager_url") && doc["manager_url"].IsString()) {
const rapidjson::Value &obj = doc["manager_url"];
manager_url_ = obj.GetString();
} else {
- manager_url_ = constants::kBusURL;
+ manager_url_ = constants::kManagerURL;
}
// bus_cluster_URL_
if (doc.HasMember("manager_cluster_url") &&
@@ -301,13 +301,13 @@ void SdkConfig::InitBusParam(const rapidjson::Value &doc) {
max_proxy_num_ = constants::kMaxBusNum;
}
- if (doc.HasMember("group_ids") && doc["group_ids"].IsString()) {
- const rapidjson::Value &obj = doc["group_ids"];
- std::string bids_str = obj.GetString();
- int32_t size = Utils::splitOperate(bids_str, group_ids_, ",");
- } else {
+ if (doc.HasMember("inlong_group_ids") && doc["inlong_group_ids"].IsString()) {
+ const rapidjson::Value &obj = doc["inlong_group_ids"];
+ std::string inlong_group_ids_str = obj.GetString();
+ Utils::splitOperate(inlong_group_ids_str, inlong_group_ids_, ",");
}
}
+
void SdkConfig::InitTcpParam(const rapidjson::Value &doc) {
if (doc.HasMember("tcp_detection_interval") &&
doc["tcp_detection_interval"].IsInt() &&
@@ -334,7 +334,29 @@ void SdkConfig::InitTcpParam(const rapidjson::Value &doc) {
tcp_idle_time_ = constants::kTcpIdleTime;
}
}
-
+void SdkConfig::InitAuthParm(const rapidjson::Value &doc) {
+ // auth settings
+ if (doc.HasMember("need_auth") && doc["need_auth"].IsBool() &&
+ doc["need_auth"].GetBool()) {
+ if (!doc.HasMember("auth_id") || !doc.HasMember("auth_key") ||
+ !doc["auth_id"].IsString() || !doc["auth_key"].IsString() ||
+ doc["auth_id"].IsNull() || doc["auth_key"].IsNull()) {
+ LOG_ERROR("need_auth, but auth_id or auth_key is empty or not string "
+ "type, check config!");
+ return;
+ }
+ need_auth_ = true;
+ auth_id_ = doc["auth_id"].GetString();
+ auth_key_ = doc["auth_key"].GetString();
+ LOG_INFO("need_auth, auth_id:%s, auth_key:%s" << auth_id_.c_str()
+ << auth_key_.c_str());
+ } else {
+ need_auth_ = constants::kNeedAuth;
+ LOG_INFO("need_auth is not expect, then use default:%s" << need_auth_
+ ? "true"
+ : "false");
+ }
+}
void SdkConfig::OthersParam(const rapidjson::Value &doc) {
// ser_ip
if (doc.HasMember("ser_ip") && doc["ser_ip"].IsString()) {
@@ -381,7 +403,7 @@ void SdkConfig::OthersParam(const rapidjson::Value &doc) {
void SdkConfig::ShowClientConfig() {
LOG_INFO("per_groupid_thread_nums: " << per_groupid_thread_nums_);
- LOG_INFO("group_ids: " << Utils::getVectorStr(group_ids_).c_str());
+ LOG_INFO("group_ids: " << Utils::getVectorStr(inlong_group_ids_).c_str());
LOG_INFO("buffer_num_per_bid: " << send_buf_size_);
LOG_INFO("local_ip: " << local_ip_.c_str());
LOG_INFO("local_port: " << local_port_);
@@ -414,5 +436,9 @@ void SdkConfig::ShowClientConfig() {
LOG_INFO("tcp_idle_time_: " << tcp_idle_time_);
LOG_INFO("send_buf_size_: " << send_buf_size_);
LOG_INFO("recv_buf_size_: " << recv_buf_size_);
+ LOG_INFO("need_auth: " << need_auth_ ? "true" : "false");
+ LOG_INFO("auth_id: " << auth_id_.c_str());
+ LOG_INFO("auth_key: " << auth_key_.c_str());
}
-} // namespace sdk
+
+} // namespace inlong
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/api_imp.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/api_imp.cc
index b6c8760f7a..0a7b8d944c 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/api_imp.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/api_imp.cc
@@ -94,8 +94,7 @@ int32_t ApiImp::CloseApi(int32_t max_waitms) {
}
int32_t ApiImp::DoInit() {
- LOG_INFO(
- "tdbus sdk cpp start Init, version:" << constants::kVersion);
+ LOG_INFO("tdbus sdk cpp start Init, version:" << constants::kVersion);
signal(SIGPIPE, SIG_IGN);
@@ -103,11 +102,11 @@ int32_t ApiImp::DoInit() {
ProxyManager::GetInstance()->Init();
- for (int i = 0; i < SdkConfig::getInstance()->group_ids_.size(); i++) {
+ for (int i = 0; i < SdkConfig::getInstance()->inlong_group_ids_.size(); i++) {
LOG_INFO("DoInit CheckBidConf inlong_group_id:"
- << SdkConfig::getInstance()->group_ids_[i]);
+ << SdkConfig::getInstance()->inlong_group_ids_[i]);
ProxyManager::GetInstance()->CheckBidConf(
- SdkConfig::getInstance()->group_ids_[i], false);
+ SdkConfig::getInstance()->inlong_group_ids_[i], false);
}
return InitManager();
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.cc
index 28edc7e413..142cbe5fbc 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.cc
@@ -80,22 +80,30 @@ void ProxyManager::DoUpdate() {
if (SdkConfig::getInstance()->enable_manager_url_from_cluster_)
url = SdkConfig::getInstance()->manager_cluster_url_;
else {
- url = SdkConfig::getInstance()->manager_url_ + "/" + groupid2cluster.first;
+ url = SdkConfig::getInstance()->manager_url_ + "/" +
+ groupid2cluster.first;
}
std::string post_data = "ip=" + SdkConfig::getInstance()->local_ip_ +
"&version=" + constants::kVersion +
"&protocolType=" + constants::kProtocolType;
- LOG_WARN("get inlong_group_id:%s proxy cfg url:%s, post_data:%s"<<
- groupid2cluster.first.c_str()<<url.c_str()<< post_data.c_str());
+ LOG_WARN("get inlong_group_id:" << groupid2cluster.first.c_str()
+ << "proxy cfg url " << url.c_str()
+ << "post_data:" << post_data.c_str());
std::string meta_data;
int32_t ret;
std::string urlByDNS;
for (int i = 0; i < constants::kMaxRequestTDMTimes; i++) {
- ret = Utils::requestUrl(url, urlByDNS, meta_data, timeout_);
+ HttpRequest request = {url,
+ timeout_,
+ SdkConfig::getInstance()->need_auth_,
+ SdkConfig::getInstance()->auth_id_,
+ SdkConfig::getInstance()->auth_key_,
+ post_data};
+ ret = Utils::requestUrl(meta_data, &request);
if (!ret) {
break;
- }
+ } // request success
}
ProxyInfoVec proxyInfoVec;
ret = ParseAndGet(groupid2cluster.first, meta_data, proxyInfoVec);
@@ -116,7 +124,7 @@ void ProxyManager::DoUpdate() {
LOG_INFO("finish ProxyManager DoUpdate.");
}
-int32_t ProxyManager::ParseAndGet(const std::string &groupid,
+int32_t ProxyManager::ParseAndGet(const std::string &inlong_group_id,
const std::string &meta_data,
ProxyInfoVec &proxy_info_vec) {
rapidjson::Document doc;
@@ -126,67 +134,84 @@ int32_t ProxyManager::ParseAndGet(const std::string &groupid,
return SdkCode::kErrorParseJson;
}
- if (doc.HasMember("size") && doc["size"].IsInt() && !doc["size"].IsNull() &&
- doc["size"].GetInt() > 0) {
- const rapidjson::Value &obj = doc["size"];
- } else {
- LOG_ERROR("can't find groupid:%s buslist from meta_data"
- << groupid.c_str());
+ if (!(doc.HasMember("success") && doc["success"].IsBool() &&
+ doc["success"].GetBool())) {
+ LOG_ERROR("failed to get proxy_list of inlong_group_id:%s, success: not "
+ "exist or false"
+ << inlong_group_id.c_str());
+ return SdkCode::kErrorParseJson;
+ }
+ // check data valid
+ if (!doc.HasMember("data") || doc["data"].IsNull()) {
+ LOG_ERROR("failed to get proxy_list of inlong_group_id:%s, data: not exist "
+ "or null"
+ << inlong_group_id.c_str());
return SdkCode::kErrorParseJson;
}
- if (doc.HasMember("cluster_id") && doc["cluster_id"].IsInt() &&
- !doc["cluster_id"].IsNull()) {
- const rapidjson::Value &obj = doc["cluster_id"];
- } else {
- LOG_ERROR("cluster_id of groupid:%s is not found or not a integer"
- << groupid.c_str());
+ // check nodelist valid
+ const rapidjson::Value &clusterInfo = doc["data"];
+ if (!clusterInfo.HasMember("nodeList") || clusterInfo["nodeList"].IsNull()) {
+ LOG_ERROR("invalid nodeList of inlong_group_id:%s, not exist or null"
+ << inlong_group_id.c_str());
return SdkCode::kErrorParseJson;
}
- if (doc.HasMember("address") && !doc["address"].IsNull()) // v2版本
- {
- const rapidjson::Value &hostlist = doc["address"];
- for (auto &info : hostlist.GetArray()) {
- std::string id, ip;
- int32_t port;
- if (info.HasMember("host") && !info["host"].IsNull())
- ip = info["host"].GetString();
- else {
- LOG_ERROR("this host info is null");
- continue;
- }
- if (info.HasMember("port") && !info["port"].IsNull()) {
- if (info["port"].IsString())
- port = std::stoi(info["port"].GetString());
- if (info["port"].IsInt())
- port = info["port"].GetInt();
- }
+ // check nodeList isn't empty
+ const rapidjson::Value &nodeList = clusterInfo["nodeList"];
+ if (nodeList.GetArray().Size() == 0) {
+ LOG_ERROR("empty nodeList of inlong_group_id:%s"
+ << inlong_group_id.c_str());
+ return SdkCode::kErrorParseJson;
+ }
+ // check clusterId
+ if (!clusterInfo.HasMember("clusterId") ||
+ !clusterInfo["clusterId"].IsInt() ||
+ clusterInfo["clusterId"].GetInt() < 0) {
+ LOG_ERROR("clusterId of inlong_group_id:%s is not found or not a integer"
+ << inlong_group_id.c_str());
+ return SdkCode::kErrorParseJson;
+ }
- else {
- LOG_ERROR("this port info is null or negative");
- continue;
- }
- if (info.HasMember("id") && !info["id"].IsNull()) {
- if (info["id"].IsString())
- id = info["id"].GetString();
- if (info["id"].IsInt())
- id = std::to_string(info["id"].GetInt());
- } else {
- LOG_ERROR("there is no id info of groupid");
- continue;
- }
- proxy_info_vec.emplace_back(id, ip, port);
+ // proxy list
+ for (auto &proxy : nodeList.GetArray()) {
+ std::string ip;
+ std::string id;
+ int32_t port;
+ if (proxy.HasMember("ip") && !proxy["ip"].IsNull())
+ ip = proxy["ip"].GetString();
+ else {
+ LOG_ERROR("this ip info is null");
+ continue;
}
- } else {
- LOG_ERROR("there is no any host info of groupid:%s" << groupid.c_str());
- return SdkCode::kErrorParseJson;
+ if (proxy.HasMember("port") && !proxy["port"].IsNull()) {
+ if (proxy["port"].IsString())
+ port = std::stoi(proxy["port"].GetString());
+ else if (proxy["port"].IsInt())
+ port = proxy["port"].GetInt();
+ }
+
+ else {
+ LOG_ERROR("this ip info is null or negative");
+ continue;
+ }
+ if (proxy.HasMember("id") && !proxy["id"].IsNull()) {
+ if (proxy["id"].IsString())
+ id = proxy["id"].GetString();
+ else if (proxy["id"].IsInt())
+ id = proxy["id"].GetInt();
+ } else {
+ LOG_WARN("there is no id info of inlong_group_id");
+ continue;
+ }
+ proxy_info_vec.emplace_back(id, ip, port);
}
+
return SdkCode::kSuccess;
}
int32_t ProxyManager::GetProxy(const std::string &groupid,
- ProxyInfoVec &proxy_info_vec) {
+ ProxyInfoVec &proxy_info_vec) {
unique_read_lock<read_write_mutex> rdlck(groupid_2_proxy_map_rwmutex_);
auto it = groupid_2_proxy_map_.find(groupid);
if (it == groupid_2_proxy_map_.end()) {
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.h
index 50a27e478b..25d78beb13 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.h
@@ -30,7 +30,7 @@ namespace inlong {
class ProxyManager {
private:
static ProxyManager *instance_;
- int32_t timeout_;
+ uint32_t timeout_;
read_write_mutex groupid_2_cluster_rwmutex_;
read_write_mutex groupid_2_proxy_map_rwmutex_;
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/send_manager.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/send_manager.cc
index 3c45537319..f883bf44d7 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/send_manager.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/send_manager.cc
@@ -22,11 +22,13 @@
#include "proxy_manager.h"
namespace inlong {
SendManager::SendManager() : send_group_idx_(0) {
- for (int32_t i = 0; i < SdkConfig::getInstance()->group_ids_.size(); i++) {
+ for (int32_t i = 0; i < SdkConfig::getInstance()->inlong_group_ids_.size();
+ i++) {
LOG_INFO("SendManager, group_id:"
- << SdkConfig::getInstance()->group_ids_[i] << " send group num:"
+ << SdkConfig::getInstance()->inlong_group_ids_[i]
+ << " send group num:"
<< SdkConfig::getInstance()->per_groupid_thread_nums_);
- DoAddSendGroup(SdkConfig::getInstance()->group_ids_[i]);
+ DoAddSendGroup(SdkConfig::getInstance()->inlong_group_ids_[i]);
}
}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/CMakeLists.txt b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/CMakeLists.txt
deleted file mode 100644
index c67e8331d5..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/CMakeLists.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-cmake_minimum_required(VERSION 3.1)
-
-aux_source_directory(. BASE_SRCS)
-add_library(sdk_net STATIC ${BASE_SRCS})
-
-target_link_libraries(sdk_net liblog4cplus.a)
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/buffer_pool.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/buffer_pool.cc
deleted file mode 100644
index 55ed55f452..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/buffer_pool.cc
+++ /dev/null
@@ -1,428 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "buffer_pool.h"
-
-#include <chrono>
-#include <memory>
-#include <vector>
-
-#include "proxylist_config.h"
-#include "sdk_core.h"
-#include "logger.h"
-#include "send_buffer.h"
-#include "socket_connection.h"
-#include "tc_api.h"
-
-namespace dataproxy_sdk
-{
- BufferPool::BufferPool(uint32_t pool_id, uint32_t buf_num, uint32_t buf_size)
- : pool_id_(pool_id), buf_num_(buf_num), read_(0), write_(0), current_use_(0), pool_mutex_(), executor_(std::make_shared<ExecutorThread>(pool_id))
- {
- buffers_.reserve(buf_num_);
- for (int i = 0; i < buf_num_; i++)
- {
- buffers_.push_back(new SendBuffer(buf_size));
- }
- }
-
- BufferPool::~BufferPool()
- {
- for (auto it : buffers_)
- {
- delete it;
- }
- buffers_.clear();
- LOG_DEBUG("free send buffer memory, pool(id:%d)", pool_id_);
- }
-
- bool BufferPool::isAvaliable()
- {
- return current_use_ < buf_num_;
- }
-
- int32_t BufferPool::getSendBuf(SendBuffer *&send_buf)
- {
- std::lock_guard<std::mutex> lck(pool_mutex_);
-
- if (current_use_ >= buf_num_)
- {
- LOG_ERROR("buffer pool(id:%d) is full", pool_id_);
- return SDKInvalidResult::kBufferPoolFull;
- }
-
- while (buffers_[write_]->isUsed())
- {
- write_ = (write_ + 1) % buf_num_;
- }
-
- send_buf = buffers_[write_];
- // ++write_;
- if (!send_buf)
- {
- LOG_ERROR("failed to get send buf, pool(id:%d)", pool_id_);
- return SDKInvalidResult::kFailGetSendBuf;
- }
-
- send_buf->setIsUsed(true);
- ++current_use_;
-
- return 0;
- }
-
- int32_t BufferPool::sendBufToConn(SendBuffer *&send_buf)
- {
- std::lock_guard<std::mutex> pool_lck(pool_mutex_);
-
- std::lock_guard<std::mutex> buf_lck(send_buf->mutex_);
-
- auto self = shared_from_this();
- if (g_config.retry_num_ > 0 && g_config.retry_interval_ > 0) //resend if need
- {
- executor_->postTask([self, this, send_buf]
- {
- send_buf->timeout_timer_ = executor_->createSteadyTimer();
- send_buf->timeout_timer_->expires_after(std::chrono::milliseconds(g_config.retry_interval_));
- send_buf->timeout_timer_->async_wait(std::bind(&BufferPool::RetryHandler, shared_from_this(), std::placeholders::_1, send_buf)); });
- }
- if (send_buf->target()->isStop())
- {
- auto new_conn = g_clusters->createActiveConn(send_buf->inlong_group_id(), pool_id_);
- if (!new_conn)
- {
- LOG_WARN("first send buf, fail to create new conn for sendbuf, inlong_group_id:%s, inlong_stream_id:%s", send_buf->inlong_group_id().c_str(), send_buf->inlong_stream_id().c_str());
- send_buf->fail_create_conn_.increment();
- // send_buf->increaseRetryNum();
- // send_err_.increment();
- already_send_buf_list_[send_buf->uniqId()] = send_buf; //preparing for next sending
- return -1;
- }
- send_buf->setTarget(new_conn);
- }
- send_buf->target()->sendBuf(send_buf); //do send operation
- send_buf->increaseRetryNum();
- LOG_TRACE("request buf(id:%d) send to connection%s first time, inlong_group_id:%s, inlong_stream_id:%s", send_buf->uniqId(), send_buf->target()->getRemoteInfo().c_str(),
- send_buf->inlong_group_id().c_str(), send_buf->inlong_stream_id().c_str());
-
- already_send_buf_list_[send_buf->uniqId()] = send_buf; //add buf uid into sent list
-
- send_total_.increment();
-
- //update metrics
- has_send_buf_.increment();
- waiting_ack_.increment();
-
- LOG_TRACE("success to write buf pool, pool(id:%d), buf(uid:%d), inlong_group_id:%s, inlong_stream_id:%s", pool_id_, send_buf->uniqId(), send_buf->inlong_group_id().c_str(),
- send_buf->inlong_stream_id().c_str());
- return 0;
- }
-
- void BufferPool::RetryHandler(const std::error_code &ec, SendBuffer *buf)
- {
- if (ec) // timer is cancelled, two cases: 1.ackbuf->sendbuf.reset;2.msg_type=2,conn.doWrite->cancel
- {
- if (g_config.msg_type_ == 2)
- {
- LOG_TRACE("msg_type is 2, no need ackmsg, clear buf(uid:%d) directly", buf->uniqId());
- ackBuf(buf->uniqId());
- }
- return;
- }
-
- if (!buf->isUsed())
- {
- return;
- } // buf is already acked before retry
-
- std::lock_guard<std::mutex> buf_lck(buf->mutex_);
-
- if (buf->getAlreadySend() == 2)
- {
- LOG_INFO("buf(id:%d, inlong_group_id:%s, inlong_stream_id:%s) ackmsg timeout, send %d times(max retry_num:%d)", buf->uniqId(), buf->inlong_group_id().c_str(),
- buf->inlong_stream_id().c_str(), buf->getAlreadySend(), g_config.retry_num_);
- }
- else
- {
- LOG_DEBUG("buf(id:%d, inlong_group_id:%s, inlong_stream_id:%s) ackmsg timeout, send %d times(max retry_num:%d)", buf->uniqId(), buf->inlong_group_id().c_str(),
- buf->inlong_stream_id().c_str(), buf->getAlreadySend(), g_config.retry_num_);
- }
-
- // max_retry_num, usercallback
- if (buf->getAlreadySend() >= g_config.retry_num_ || buf->fail_create_conn_.get() >= constants::kMaxRetryConnection)
- {
- LOG_WARN("fail to send buf(id:%d, inlong_group_id:%s, inlong_stream_id:%s), has send max_retry_num(%d) times, start usercallback", buf->uniqId(), buf->inlong_group_id().c_str(),
- buf->inlong_stream_id().c_str(), g_config.retry_num_);
- buf->doUserCallBack();
- buf->reset();
- }
- else // ack timeout, resend
- {
- if (!buf->target() || buf->target()->isStop())
- {
- auto new_conn = g_clusters->createActiveConn(buf->inlong_group_id(), pool_id_); // TODO: should improve as choosing from active conn instread of creating?
-
- if (!new_conn) //create conn error, waiting for next creating
- {
- LOG_INFO("fail to create new conn to send buf, inlong_group_id:%s, inlong_stream_id:%s", buf->inlong_group_id().c_str(), buf->inlong_stream_id().c_str());
- buf->fail_create_conn_.increment();
- buf->timeout_timer_->expires_after(std::chrono::milliseconds(g_config.retry_interval_));
- buf->timeout_timer_->async_wait(std::bind(&BufferPool::RetryHandler, shared_from_this(), std::placeholders::_1, buf));
- return;
- }
- buf->setTarget(new_conn);
- }
-
- buf->target()->sendBuf(buf);
- buf->increaseRetryNum();
- buf->timeout_timer_->expires_after(std::chrono::milliseconds(g_config.retry_interval_));
- buf->timeout_timer_->async_wait(std::bind(&BufferPool::RetryHandler, shared_from_this(), std::placeholders::_1, buf));
- }
- }
-
- void BufferPool::ackBufHelper(uint32_t uniq_id)
- {
- std::lock_guard<std::mutex> lck(pool_mutex_);
- if (already_send_buf_list_.find(uniq_id) == already_send_buf_list_.end())
- {
- LOG_ERROR("no buf(uid:%d) in already_send_buf_list", uniq_id);
- return;
- }
- auto &buf2ack = already_send_buf_list_[uniq_id];
-
- std::lock_guard<std::mutex> buf_lck(buf2ack->mutex_);
-
- int32_t msg_cnt = buf2ack->msgCnt();
- success_ack_.add(msg_cnt);
-
- //update packqueue metrics
- if(g_queues){
- auto packqueue = g_queues->getPackQueue(buf2ack->inlong_group_id(), buf2ack->inlong_stream_id());
- packqueue->success_num_.add(msg_cnt);
- packqueue->total_success_num_.add(msg_cnt);
- }
-
- already_send_buf_list_[uniq_id]->reset();
- already_send_buf_list_.erase(uniq_id);
- --current_use_;
-
- waiting_ack_.decrement();
- has_ack_.increment();
- LOG_TRACE("pool(id:%d) success ack msg cumulative num: %d", pool_id_, success_ack_.get());
- }
-
- void BufferPool::showState(const std::string &inlong_group_id)
- {
- if (inlong_group_id.empty())
- {
- LOG_STAT("STATE|pool_id:%d, current_use:%d(total:%d), invoke_send_buf:%d, has_ack:%d, waiting_ack:%d", pool_id_, currentUse(),
- g_config.bufNum(), has_send_buf_.get(), has_ack_.get(), waiting_ack_.get());
- }
- else
- {
- LOG_STAT("STATE|inlong_group_id:%s, pool_id:%d, current_use:%d(total:%d), invoke_send_buf:%d, has_ack:%d, waiting_ack:%d", inlong_group_id.c_str(), pool_id_, currentUse(),
- g_config.bufNum(), has_send_buf_.get(), has_ack_.get(), waiting_ack_.get());
- }
- }
-
- void BufferPool::close()
- {
- if (executor_ != nullptr)
- {
- executor_->close();
- }
- executor_.reset();
- }
-
- TotalPools::TotalPools() : next_(0), mutex_()
- {
- if (g_config.enable_groupId_isolation_) // different groupid data use different bufpool
- {
- for (int32_t i = 0; i < g_config.inlong_group_ids_.size(); i++) // create a bufpool for ervery groupidS
- {
- std::vector<BufferPoolPtr> groupid_pool;
- groupid_pool.reserve(g_config.buffer_num_per_groupId_);
- for (int32_t j = 0; j < g_config.buffer_num_per_groupId_; j++)
- {
- groupid_pool.push_back(std::make_shared<BufferPool>(j, g_config.bufNum(), g_config.buf_size_));
- }
-
- groupid2pool_map_[g_config.inlong_group_ids_[i]] = groupid_pool;
- groupid2next_[g_config.inlong_group_ids_[i]] = 0;
- }
- }
- else // round-robin
- {
- pools_.reserve(g_config.shared_buf_nums_);
- for (int i = 0; i < g_config.shared_buf_nums_; i++)
- {
- pools_.push_back(std::make_shared<BufferPool>(i, g_config.bufNum(), g_config.buf_size_));
- }
- }
- }
-
- BufferPoolPtr TotalPools::getPool(const std::string &inlong_group_id)
- {
- if (g_config.enable_groupId_isolation_) // groupid isolate
- {
- auto groupid_pool = groupid2pool_map_.find(inlong_group_id);
- if (groupid_pool == groupid2pool_map_.end() || groupid_pool->second.empty())
- {
- LOG_ERROR("fail to get bufferpool, inlong_group_id:%s", inlong_group_id.c_str());
- return nullptr;
- }
- if (groupid2next_.find(inlong_group_id)==groupid2next_.end())
- {
- groupid2next_[inlong_group_id]=0;
- }
-
- auto& pool_set=groupid_pool->second;
- int32_t idx=0;
- for (int32_t i = 0; i < pool_set.size(); i++)
- {
- idx = (groupid2next_[inlong_group_id]++) % pool_set.size();
- if (pool_set[idx]->isAvaliable())
- {
- return pool_set[idx];
- }
-
- }
-
- return nullptr;
- }
- else
- {
- if (pools_.empty())
- {
- LOG_ERROR("fail to get bufferpool, allocate error in tc_init");
- return nullptr;
- }
-
- int32_t idx=0;
- for (int32_t i = 0; i < pools_.size(); i++)
- {
- idx = (next_++) % pools_.size();
- if (pools_[idx]->isAvaliable())
- {
- return pools_[idx];
- }
-
- }
-
- return nullptr;
-
- }
- }
-
- bool TotalPools::isPoolAvailable(const std::string &inlong_group_id)
- {
-
- if (g_config.enable_groupId_isolation_) // groupid_isolation
- {
- auto groupid_pool = groupid2pool_map_.find(inlong_group_id);
- if (groupid_pool == groupid2pool_map_.end())
- {
- LOG_ERROR("no buffer allocated to inlong_group_id:%s, check config", inlong_group_id.c_str());
- return false;
- }
- for (int i = 0; i < groupid_pool->second.size(); i++)
- {
- if (groupid_pool->second[i]->isAvaliable())
- {
- return true;
- }
- }
- return false;
- }
- else // rr
- {
- for (int i = 0; i < pools_.size(); i++)
- {
- if (pools_[i]->isAvaliable())
- {
- return true;
- }
- }
- return false;
- }
- }
-
- void TotalPools::addUid2BufPool(uint32_t uniqId, BufferPoolPtr& bpr){
- std::lock_guard<std::mutex> lck(mutex_);
- uid2buf_pool_[uniqId]=bpr;
- }
-
- BufferPoolPtr TotalPools::getUidBufPool(uint32_t uniqId){
- std::lock_guard<std::mutex> lck(mutex_);
- auto iter=uid2buf_pool_.find(uniqId);
- if(iter==uid2buf_pool_.end()){
- return nullptr;
- }
- uid2buf_pool_.erase(iter); //TODO: need add it into bufpool after ack operation
-
- return iter->second;
-
- }
-
- void TotalPools::showState()
- {
- if (g_config.enable_groupId_isolation_)
- {
- for (auto &groupid_pool : groupid2pool_map_)
- {
- showStateHelper(groupid_pool.first, groupid_pool.second);
- }
- }
- else
- {
- showStateHelper("", pools_);
- }
- }
-
- void TotalPools::showStateHelper(const std::string &inlong_group_id, std::vector<BufferPoolPtr> &pools)
- {
- for (auto &pool : pools)
- {
- pool->showState(inlong_group_id);
- }
- }
-
- void TotalPools::close()
- {
- // waiting for sending the rest data
- if (uid2buf_pool_.size())
- {
- LOG_INFO("waiting for 10s to ack remaining msg");
- std::this_thread::sleep_for(std::chrono::seconds(10));
- }
-
- for (auto &pool : pools_)
- {
- pool->close();
- }
-
- for (auto &groupid_pool : groupid2pool_map_)
- {
- for (auto &pool : groupid_pool.second)
- {
- pool->close();
- }
- }
- }
-
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/buffer_pool.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/buffer_pool.h
deleted file mode 100644
index dc48cdcaf1..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/buffer_pool.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_NET_BUFFER_POOL_H_
-#define DATAPROXY_SDK_NET_BUFFER_POOL_H_
-
-#include <deque>
-#include <mutex>
-#include <system_error>
-#include <thread>
-#include <unordered_map>
-#include <vector>
-
-#include "sdk_core.h"
-#include "noncopyable.h"
-#include "pack_queue.h"
-#include "recv_buffer.h"
-#include "executor_thread_pool.h"
-
-namespace dataproxy_sdk
-{
- class SendBuffer;
- class BufferPool : noncopyable, public std::enable_shared_from_this<BufferPool>
- {
- private:
- uint32_t pool_id_;
- uint32_t buf_num_; //sendbuf count
- uint32_t read_, write_;
- int32_t current_use_;//sendbuf being used count
- std::vector<SendBuffer *> buffers_; // allocate memory
- std::unordered_map<uint32_t, SendBuffer *> already_send_buf_list_; //key:uniq_id_, sent but not ack
- mutable std::mutex pool_mutex_;
- ExecutorThreadPtr executor_;
-
- AtomicInt send_total_; //send buf num, including resend
- AtomicInt send_err_; //fail send buf num, including resend
- AtomicInt resend_total_; //total resent package
- AtomicInt success_ack_; //total ack msg
-
- public:
- // bufusage metrics
- AtomicUInt waiting_ack_;
- AtomicUInt has_ack_;
- AtomicUInt has_send_buf_; //sent to conn
-
- public:
- BufferPool(uint32_t pool_id, uint32_t buf_num, uint32_t buf_size);
- virtual ~BufferPool();
-
- bool isAvaliable(); //is bufpool avaliable
- virtual int32_t getSendBuf(SendBuffer *&send_buf);
- int32_t sendBufToConn(SendBuffer *&send_buf);
-
- void ackBufHelper(uint32_t uinq_id); // ack
- void ackBuf(uint32_t uniq_id) { executor_->postTask(std::bind(&BufferPool::ackBufHelper, shared_from_this(), uniq_id)); }
-
- uint32_t poolId() const { return pool_id_; }
-
- uint32_t writeId() const { return write_; }
-
- int32_t currentUse() const { return current_use_; }
-
- void showState(const std::string &inlong_group_id);
-
- void close();
-
- private:
- void RetryHandler(const std::error_code &ec, SendBuffer *buf);
- };
-
- using BufferPoolPtr = std::shared_ptr<BufferPool>;
-
- class TotalPools : noncopyable
- {
- private:
- // round-robin
- std::vector<BufferPoolPtr> pools_;
- int32_t next_;
-
- // groupid_isolation
- std::unordered_map<std::string, std::vector<BufferPoolPtr>> groupid2pool_map_;
- std::unordered_map<std::string, int32_t> groupid2next_;
-
- std::unordered_map<uint32_t, BufferPoolPtr> uid2buf_pool_;// <sent buf uid, bufpool>
-
- mutable std::mutex mutex_;
-
- void close(); // shutdown buffer thread
- void showStateHelper(const std::string &inlong_group_id, std::vector<BufferPoolPtr> &pools);
-
- public:
- TotalPools();
- virtual ~TotalPools() { close(); } // FIXME: need other operations?
- bool isPoolAvailable(const std::string &inlong_group_id);
- virtual BufferPoolPtr getPool(const std::string &inlong_group_id);
- void addUid2BufPool(uint32_t uniqId, BufferPoolPtr& bpr);
- BufferPoolPtr getUidBufPool(uint32_t uniqId);
- void showState();
- };
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_NET_BUFFER_POOL_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/executor_thread_pool.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/executor_thread_pool.cc
deleted file mode 100644
index 4d6068f60a..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/executor_thread_pool.cc
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "executor_thread_pool.h"
-
-namespace dataproxy_sdk
-{
- ExecutorThread::ExecutorThread(int32_t id)
- : io_context_(std::make_shared<asio::io_context>()), work_(asio::make_work_guard(*io_context_)), worker_(std::bind(&ExecutorThread::startWorker, this)), thread_id_(id)
- {
- }
-
- ExecutorThread::~ExecutorThread()
- {
- LOG_DEBUG("thread:%d destructor", thread_id_);
- close();
- if (worker_.joinable())
- {
- worker_.detach();
- }
- }
-
- void ExecutorThread::close()
- {
- LOG_DEBUG("thread:%d close() --> io_context stop", thread_id_);
- io_context_->stop();
- if (std::this_thread::get_id() != worker_.get_id() && worker_.joinable())
- {
- LOG_DEBUG("join thread %d", thread_id_);
- worker_.join();
- }
- }
-
- void ExecutorThread::startWorker() //start listen
- {
- LOG_DEBUG("thread:%d start io_context run", thread_id_);
- io_context_->run();
- }
-
- TcpSocketPtr ExecutorThread::createTcpSocket() //create socket based on io_context
- {
- return std::make_shared<asio::ip::tcp::socket>(*io_context_);
- }
-
- SteadyTimerPtr ExecutorThread::createSteadyTimer() //create timer based on io_context
- {
- return std::make_shared<asio::steady_timer>(*io_context_);
- }
-
- ExecutorThreadPool::ExecutorThreadPool() : next_idx_(0)
- {
- for (int i = 0; i < g_config.thread_nums_; i++)
- {
- executors_.emplace_back(std::make_shared<ExecutorThread>(i));
- }
-
- }
-
- ExecutorThreadPtr ExecutorThreadPool::nextExecutor()
- {
- if (executors_.empty())
- {
- LOG_ERROR("fail to find executor thread pool");
- return nullptr;
- }
- int32_t idx = (next_idx_++) % executors_.size();
- return executors_[idx];
- }
-
- ExecutorThreadPtr ExecutorThreadPool::getExecutor(int32_t id)
- {
- if (id < 0 || id >= executors_.size() || executors_.empty())
- {
- LOG_ERROR("fail to get network_thread<id:%d>, max id is %d", id, executors_.size() - 1);
- return nullptr;
- }
- return executors_[id];
-
- }
-
- void ExecutorThreadPool::showState()
- {
- for (auto &executor : executors_)
- {
- executor->showState();
- }
-
- }
-
- void ExecutorThreadPool::close()
- {
- for (auto it : executors_)
- {
- if (it != nullptr)
- {
- it->close();
- LOG_DEBUG("thread_id:%d, shared_count:%d", it->threadId(), it.use_count());
- }
- it.reset();
- }
- executors_.clear();
-
- }
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/executor_thread_pool.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/executor_thread_pool.h
deleted file mode 100644
index bfc5d74f70..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/executor_thread_pool.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef CPAI_NET_EXECUTOR_THREAD_POOL_H_
-#define CPAI_NET_EXECUTOR_THREAD_POOL_H_
-
-#include <asio.hpp>
-#include <functional>
-#include <memory>
-#include <stdint.h>
-#include <thread>
-
-#include "sdk_core.h"
-#include "logger.h"
-#include "noncopyable.h"
-
-namespace dataproxy_sdk
-{
- class ExecutorThread : noncopyable
- {
- private:
- std::shared_ptr<asio::io_context> io_context_;
- io_context_work work_; //guarantee io_context.run() don't exit
- std::thread worker_;
- int32_t thread_id_;
-
- public:
- AtomicUInt waiting_send_;
-
- public:
- ExecutorThread(int32_t id);
- virtual ~ExecutorThread();
-
- inline void postTask(std::function<void(void)> task) { io_context_->post(task); }
- std::shared_ptr<asio::io_context> getIoContext() { return io_context_; }
- TcpSocketPtr createTcpSocket();
- SteadyTimerPtr createSteadyTimer();
- inline void showState(){LOG_DEBUG("------->thread_id:%d, waiting_send:%d", thread_id_, waiting_send_.get());}
- void close();
-
- int32_t threadId() const { return thread_id_; }
-
- private:
- void startWorker();
- };
-
- //network threads
- class ExecutorThreadPool : noncopyable
- {
- private:
- // round-robin
- std::vector<ExecutorThreadPtr> executors_;
- int32_t next_idx_;
-
- public:
- explicit ExecutorThreadPool();
- virtual ~ExecutorThreadPool() { close(); }
-
- ExecutorThreadPtr nextExecutor(); // round-robbin, get executor thread, for new conn creating
- ExecutorThreadPtr getExecutor(int32_t id);
- void showState();
- void close();
- };
-
-} // namespace dataproxy_sdk
-
-#endif // CPAI_NET_EXECUTOR_THREAD_POOL_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/send_buffer.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/send_buffer.h
deleted file mode 100644
index 2699eb4622..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/send_buffer.h
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_NET_SEND_BUFFER_H_
-#define DATAPROXY_SDK_NET_SEND_BUFFER_H_
-
-#include <mutex>
-#include <string>
-
-#include "sdk_core.h"
-// #include "executor_thread_pool.h"
-#include "logger.h"
-#include "noncopyable.h"
-// #include "socket_connection.h"
-
-namespace dataproxy_sdk
-{
-class SendBuffer : noncopyable
-{
- private:
- uint32_t uniq_id_;
- bool is_used_;
- bool is_packed_; //is packed completed
- char* content_; //send data
- uint32_t size_; //buf_size
- int32_t msg_cnt_;
- uint32_t len_; //send data len
- std::string inlong_group_id_;
- std::string inlong_stream_id_;
- AtomicInt already_send_;
- uint64_t first_send_time_; //ms
- uint64_t latest_send_time_; //ms
- ConnectionPtr target_; //send conn
- std::vector<UserMsgPtr> user_msg_set_;
-
- public:
- std::mutex mutex_;
- SteadyTimerPtr timeout_timer_; //timeout, resend
- AtomicInt fail_create_conn_; //create conn fail count
-
- public:
- SendBuffer(uint32_t size)
- : is_used_(false)
- , is_packed_(false)
- , mutex_()
- , msg_cnt_(0)
- , len_(0)
- , inlong_group_id_()
- , inlong_stream_id_()
- , first_send_time_(0)
- , latest_send_time_(0)
- , target_(nullptr)
- , uniq_id_(0)
- , timeout_timer_(nullptr)
- , size_(size)
- {
- content_ = new char[size];
- if (content_)
- {
- memset(content_, 0x0, size);
- }
- }
- ~SendBuffer()
- {
- if (content_) { delete[] content_; }
- content_ = nullptr;
- }
-
- char* content() { return content_; }
- int32_t msgCnt() const { return msg_cnt_; }
- void setMsgCnt(const int32_t& msg_cnt) { msg_cnt_ = msg_cnt; }
- uint32_t len() { return len_; }
- void setLen(const uint32_t len) { len_ = len; }
- std::string inlong_group_id() { return inlong_group_id_; }
- std::string inlong_stream_id() { return inlong_stream_id_; }
- void setGroupid(const std::string& inlong_group_id) { inlong_group_id_ = inlong_group_id; }
- void setStreamid(const std::string& inlong_stream_id) { inlong_stream_id_ = inlong_stream_id; }
- uint64_t firstSendTime() const { return first_send_time_; }
- void setFirstSendTime(const uint64_t& first_send_time) { first_send_time_ = first_send_time; }
- uint64_t latestSendTime() const { return latest_send_time_; }
- void setLatestSendTime(const uint64_t& latest_send_time) { latest_send_time_ = latest_send_time; }
-
- ConnectionPtr target() const { return target_; }
- void setTarget(ConnectionPtr& target) { target_ = target; }
-
- inline void increaseRetryNum() { already_send_.increment(); }
- inline int32_t getAlreadySend() { return already_send_.get(); }
-
- uint32_t uniqId() const { return uniq_id_; }
- void setUniqId(const uint32_t& uniq_id) { uniq_id_ = uniq_id; }
-
- void addUserMsg(UserMsgPtr u_msg) { user_msg_set_.push_back(u_msg); }
- void doUserCallBack()
- {
- LOG_TRACE("failed to send msg, start user call_back");
- for (auto it : user_msg_set_)
- {
- if (it->cb) { it->cb(inlong_group_id_.data(), inlong_stream_id_.data(), it->msg.data(), it->msg.size(), it->user_report_time, it->user_client_ip.data()); }
- }
- }
-
- void reset()
- {
- uint32_t record_uid = uniq_id_; // for debug
-
- uniq_id_ = 0;
- is_used_ = false;
- is_packed_ = false;
- memset(content_, 0x0, size_);
- msg_cnt_ = 0;
- len_ = 0;
- inlong_group_id_ = "";
- inlong_stream_id_ = "";
- already_send_.getAndSet(0);
- first_send_time_ = 0;
- latest_send_time_ = 0;
- target_ = nullptr;
- if (timeout_timer_)
- {
- timeout_timer_->cancel();
- timeout_timer_ = nullptr;
- }
- user_msg_set_.clear();
- fail_create_conn_.getAndSet(0);
- LOG_TRACE("reset senfbuf(uid:%d) successfully", record_uid);
- }
-
- bool isPacked() const { return is_packed_; }
- void setIsPacked(bool is_packed) { is_packed_ = is_packed; }
-
- bool isUsed() const { return is_used_; }
- void setIsUsed(bool is_used) { is_used_ = is_used; }
-};
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_NET_SEND_BUFFER_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/socket_connection.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/socket_connection.cc
deleted file mode 100644
index 307ca2eaa1..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/socket_connection.cc
+++ /dev/null
@@ -1,521 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "socket_connection.h"
-
-#include <chrono>
-#include <mutex>
-#include <stdint.h>
-
-#include "buffer_pool.h"
-#include "proxylist_config.h"
-#include "sdk_constant.h"
-#include "send_buffer.h"
-#include "utils.h"
-
-namespace dataproxy_sdk
-{
- Connection::Connection(ExecutorThreadPtr &executor, ProxyInfoPtr &proxyinfo)
- : executor_(executor), thread_id_(executor_->threadId()), socket_(std::move(executor_->createTcpSocket())), status_(kConnecting), timer_(std::move(executor->createSteadyTimer())), proxyinfo_(proxyinfo), remote_info_(proxyinfo_->getString()), recv_buf_(std::make_shared<RecvBuffer>()), loads_(30), total_send_(0), total_read_(0), waiting_send_(0), send_err_nums_(0), next_load_idx_(0)
- {
- asio::ip::tcp::endpoint ep(asio::ip::address::from_string(proxyinfo_->ip()), static_cast<uint16_t>(proxyinfo_->portNum()));
- doConnect(ep);
- }
-
- Connection::~Connection() {}
-
- void Connection::doConnect(const asio::ip::tcp::endpoint &ep)
- {
- if (isStop())
- {
- LOG_WARN("fail to doConnect %s, connection status is disconnected", remote_info_.c_str());
- return;
- }
- status_ = kConnecting;
- timer_->expires_after(std::chrono::milliseconds(kConnectTimeout));
- timer_->async_wait(std::bind(&Connection::connectHandler, this, std::placeholders::_1));
- if (g_config.enable_TCP_nagle_ == false)
- {
- socket_->set_option(asio::ip::tcp::no_delay(true));
- } // close nagle
- socket_->async_connect(ep, [this](const std::error_code &ec)
- {
- timer_->cancel();
- if (ec)
- {
- status_ = kDisconnected;
- LOG_ERROR("%s async connect error: %s,%s", remote_info_.c_str(), ec.message().c_str(), ec.category().name());
- doClose();
- return;
- }
- status_ = kConnected;
- socket_->set_option(asio::ip::tcp::no_delay(true));
- setLocalInfo();
- LOG_INFO("l:%s->r:%s is connected", local_info_.c_str(), remote_info_.c_str());
-
- doRead();
- doWrite(); });
- }
-
- void Connection::sendBuf(SendBuffer *buf)
- {
- auto self = shared_from_this();
- executor_->postTask([self, this, buf]()
- {
- if (std::find(write_queue_.begin(), write_queue_.end(), buf) != write_queue_.end())
- {
- LOG_ERROR("send_buf (uid:%d) is repeat in connection write_queue", buf->uniqId());
- return;
- }
-
- bool queue_empty = write_queue_.empty();
- write_queue_.push_back(buf);
- waiting_send_.increment();
- executor_->waiting_send_.increment();
- LOG_TRACE("send_buf(uid:%d) is added to connection send queue, conn r:%s", buf->uniqId(), remote_info_.c_str());
- if (isConnected() && queue_empty) { doWrite(); } });
- }
-
- void Connection::sendHB(bool isBinHB)
- {
- if (isStop())
- {
- return;
- }
- if (!write_queue_.empty())
- {
- LOG_INFO("conn l:%s->r:%s has cache msg to send, heartbeat will try next time", local_info_.c_str(), remote_info_.c_str());
- return;
- }
- binHB_.total_len = htonl(sizeof(BinaryHB) - 4);
- binHB_.msg_type = 8;
- binHB_.data_time = htonl(static_cast<uint32_t>(Utils::getCurrentMsTime() / 1000));
- binHB_.body_ver = 1;
- binHB_.body_len = 0;
- binHB_.attr_len = 0;
- binHB_.magic = htons(constants::kBinaryMagic);
-
- char *hb;
- uint32_t hb_len = 0;
- if (isBinHB)
- {
- hb_len = sizeof(binHB_);
- hb = (char *)&binHB_;
- }
- else
- {
- hb_len = sizeof(msgHB);
- hb = (char *)msgHB;
- }
- LOG_DEBUG("conn l:%s->r:%s send %s", local_info_.c_str(), remote_info_.c_str(), isBinHB ? "binaryHB" : "msgHB");
- auto self = shared_from_this();
- asio::async_write(*socket_, asio::buffer(hb, hb_len), [self, this, hb_len, isBinHB](const std::error_code &ec, std::size_t reslen)
- {
- if (!ec && reslen == hb_len) // send success
- {
- LOG_DEBUG("conn l:%s->r:%s send %s successfully", local_info_.c_str(), remote_info_.c_str(), isBinHB ? "binaryHB" : "msgHB");
- send_err_nums_.getAndSet(0);
- retry_hb_.increment();
- }
- else
- {
- send_err_nums_.increment();
- LOG_ERROR("conn l:%s->r:%s send heartbeat error, this conn send_error_num:%d, error message:%s", local_info_.c_str(),
- remote_info_.c_str(), send_err_nums_.get(), ec.message().c_str());
- }
-
- if (retry_hb_.get() > g_config.retry_num_) //close and create new conn
- {
- LOG_ERROR("conn l:%s->r:%s send_error_num:%d, more than max_retry_num:%d, this conn will close", local_info_.c_str(),
- remote_info_.c_str(), retry_hb_.get(), g_config.retry_num_);
- doClose(&ec);
- } });
- }
-
- void Connection::connClose()
- {
- auto self = shared_from_this();
- executor_->postTask([self, this]()
- { doClose(); });
- LOG_DEBUG("post close request: conn l:%s->r:%s", local_info_.c_str(), remote_info_.c_str());
- }
-
- void Connection::doClose(const std::error_code *err)
- {
- if (isStop())
- {
- return;
- }
-
- status_ = kDisconnected;
- LOG_WARN("close conn, l:%s->r:%s", local_info_.c_str(), remote_info_.c_str());
- socket_->close();
-
- //clean
- write_queue_.clear();
- recv_buf_->Reset();
- }
-
- void Connection::doWrite()
- {
- if (isStop())
- {
- return;
- }
- if (write_queue_.empty())
- {
- return;
- }
-
- auto self = shared_from_this();
- auto curBuf = write_queue_.front();
-
- std::lock_guard<std::mutex> buf_lck(curBuf->mutex_);
- asio::async_write(*socket_, asio::buffer(curBuf->content(), curBuf->len()), [self, this, curBuf](const std::error_code &ec, std::size_t length)
- {
- write_queue_.pop_front();
-
- if (!ec) //send success
- {
-
- { // lock sendbuf
- std::lock_guard<std::mutex> buf_lck(curBuf->mutex_);
-
- total_send_.increment();
- waiting_send_.decrement();
- executor_->waiting_send_.decrement();
- send_err_nums_.getAndSet(0);
- LOG_TRACE("l:%s->r:%s async write data success(len:%d), content_len:%d, buf_id:%d", local_info_.c_str(), remote_info_.c_str(),
- length, curBuf->len(), curBuf->uniqId());
-
- //ack buf
- if (g_config.msg_type_ == 2) { curBuf->timeout_timer_->cancel(); }
-
- }
-
- doWrite();
- }
- else
- {
- std::lock_guard<std::mutex> buf_lck(curBuf->mutex_);
- send_err_nums_.increment();
- LOG_ERROR("l:%s->r:%s async write data error, buf_id:%d, error message:%s", local_info_.c_str(), remote_info_.c_str(), curBuf->uniqId(),
- ec.message().c_str());
- doClose(&ec);
-
- auto new_conn = g_clusters->createActiveConn(curBuf->inlong_group_id(), thread_id_);
- if (!new_conn) { curBuf->fail_create_conn_.increment(); }
- curBuf->setTarget(new_conn);
- return;
- } });
- }
-
- void Connection::doRead()
- {
- if (isStop())
- {
- return;
- }
-
- if (recv_buf_->length() == 0)
- recv_buf_->Reset();
-
- recv_buf_->EnsureWritableBytes(1024);
- auto self = shared_from_this();
- socket_->async_receive(
- asio::buffer(recv_buf_->WriteBegin(), recv_buf_->WritableBytes()), [self, this](const std::error_code &ec, std::size_t len)
- {
- if (ec)
- {
- LOG_ERROR("async read data error, l:%s->r:%s, error message:%s", local_info_.c_str(), remote_info_.c_str(), ec.message().c_str());
- doClose(&ec);
- return;
- }
- if (len == 0)
- {
- LOG_ERROR("async read 0 bytes, l:%s->r:%s", local_info_.c_str(), remote_info_.c_str());
- doClose(&ec);
- return;
- }
- recv_buf_->WriteBytes(len);
-
- //read the rest content
- std::error_code tmp_error;
- size_t left_size = socket_->available(tmp_error);
- if (left_size > 0 && !tmp_error)
- {
- recv_buf_->EnsureWritableBytes(left_size);
- size_t rlen = socket_->receive(asio::buffer(recv_buf_->WriteBegin(), left_size));
- if (rlen > 0) { recv_buf_->WriteBytes(rlen); }
- }
- LOG_TRACE("conn read %d bytes, l:%s->r:%s", recv_buf_->length(), local_info_.c_str(), remote_info_.c_str());
- // parse ack package
- doParse();
- doRead(); });
- }
-
- void Connection::connectHandler(const std::error_code &ec)
- {
- if (ec)
- return;
- if (isStop())
- return;
- LOG_ERROR("connect timeout, %s", remote_info_.c_str());
- doClose();
- }
-
- int32_t Connection::doParse()
- {
- while (true)
- {
- if (recv_buf_->length() < 5)
- {
- return 0;
- }
-
- if (recv_buf_->length() < recv_buf_->PeekUint32() + 4)
- {
- return 0;
- }
- //read ack package
- uint32_t total_len = recv_buf_->ReadUint32();
- uint8_t msg_type = recv_buf_->ReadUint8();
- if (msg_type == 3 || msg_type == 5 || msg_type == 6 || (msg_type & 0x1F) == 7)
- {
- if ((msg_type & 0x1F) != 7)
- {
- bool ret = parseProtocolAck(total_len);
- LOG_TRACE("parseProtocolAck success? %d, %s", ret, remote_info_.c_str());
- }
- else
- {
- bool ret = parseBinaryAck(total_len);
- LOG_TRACE("parseBinaryAck success? %d, %s", ret, remote_info_.c_str());
- }
- }
- else if (msg_type == 1 && total_len == 0x1)
- {
- retry_hb_.getAndSet(0);
- LOG_TRACE("success to parse a msghb_ack from %s", remote_info_.c_str());
- }
- else if (msg_type == 8) //binary hb
- {
- retry_hb_.getAndSet(0);
- bool ret = parseBinaryHB(total_len);
- LOG_TRACE("parseBinaryHB success? %d,%s", ret, remote_info_.c_str());
- }
- else
- {
- //wrong msg_type
- LOG_ERROR("parse ack, and get wrong msgtype: %d, proxy info%s", msg_type, remote_info_.c_str());
- return 1;
- // FIXME: need add other handler, such as close conn?
- }
- }
- return 0;
- }
-
- bool Connection::parseProtocolAck(uint32_t total_len)
- {
- uint32_t body_len = recv_buf_->ReadUint32();
- if (body_len > recv_buf_->length())
- {
- LOG_ERROR("body_len is %d, more than recv_buf left len:%d ", body_len, recv_buf_->length());
- if (total_len < 4)
- {
- LOG_ERROR("total_len is less than 4, this should be check");
- }
- recv_buf_->Skip(total_len - 4);
- return false;
- }
- recv_buf_->Skip(body_len);
- LOG_TRACE("body_len is %d, and skip body");
- uint32_t attr_len = recv_buf_->ReadUint32();
- if (attr_len > constants::kMaxAttrLen) {
- LOG_ERROR("attr_len(%d) > kMaxAttrLen(%d)", attr_len, constants::kMaxAttrLen);
- return false;
- }
- char attr[attr_len + 1];
- memset(attr, 0x0, attr_len + 1);
- strncpy(attr, recv_buf_->data(), attr_len);
- recv_buf_->Skip(attr_len);
- LOG_TRACE("attr_len is %d, attr info: %s", attr_len, attr);
-
- int errorCode = parseErrorCode(attr, attr_len);
- if (errorCode != 0) {
- LOG_ERROR("parseErrorCode is:%d ", errorCode);
- return false;
- }
- uint32_t buf_uniqId = parseAttr(attr, attr_len);
-
- auto bpr=g_pools->getUidBufPool(buf_uniqId);
- if(bpr!=nullptr){
- bpr->ackBuf(buf_uniqId);
- return true;
- }
-
- return false;
- }
-
- bool Connection::parseBinaryAck(uint32_t total_len)
- {
- uint32_t uniq = recv_buf_->ReadUint32();
- uint16_t attr_len = recv_buf_->ReadUint16();
- if (attr_len > constants::kMaxAttrLen) {
- LOG_ERROR("attr_len(%d) > kMaxAttrLen(%d)", attr_len, constants::kMaxAttrLen);
- return false;
- }
- char attr[attr_len + 1];
- memset(attr, 0x0, attr_len + 1);
- strncpy(attr, recv_buf_->data(), attr_len);
-
- recv_buf_->Skip(attr_len);
- uint16_t magic = recv_buf_->ReadUint16();
-
- if (total_len + 4 != 13 + attr_len)
- {
- LOG_ERROR("failed to parse binary ack, total_len(%d) + 4 != attr_len(%d) + 13", total_len, attr_len);
- return false;
- }
- if (magic != constants::kBinaryMagic)
- {
- LOG_ERROR("failed to parse binary ack, get error magic: %d", magic);
- return false;
- }
-
- int errorCode = parseErrorCode(attr, attr_len);
- if (errorCode != 0) {
- LOG_ERROR("parseErrorCode is:%d ", errorCode);
- return false;
- }
-
- auto bpr=g_pools->getUidBufPool(uniq);
- if(bpr!=nullptr){
- bpr->ackBuf(uniq);
- }
-
- return true;
- }
-
- bool Connection::parseBinaryHB(uint32_t total_len)
- {
- uint32_t data_time = recv_buf_->ReadUint32();
- uint8_t body_ver = recv_buf_->ReadUint8();
- uint32_t body_len = recv_buf_->ReadUint32();
- uint16_t load = recv_buf_->PeekUint16(); // proxy load
- recv_buf_->Skip(body_len);
- uint16_t attr_len = recv_buf_->ReadUint16();
- recv_buf_->Skip(attr_len);
- uint16_t magic = recv_buf_->ReadUint16();
-
- if (total_len + 4 != 18 + attr_len + body_len)
- {
- LOG_ERROR("failed to parse binary heartbeat ack, total_len(%d) + 4 != 18 + attr_len(%d) + body_len(%d)", total_len, attr_len, body_len);
- return false;
- }
-
- if (magic != constants::kBinaryMagic)
- {
- LOG_ERROR("failed to parse binary heartbeat ack, get error magic: %d", magic);
- return false;
- }
-
- std::lock_guard<std::mutex> lck(load_mutex_);
- if (body_ver == 1 && body_len == 2)
- {
- loads_[next_load_idx_ % 30] = load;
- LOG_TRACE("update proxy%s load, cur_load:%d, cur_idx:%d", remote_info_.c_str(), load, next_load_idx_);
- }
- else
- {
- loads_[next_load_idx_ % 30] = 0;
- LOG_TRACE("update proxy%s loads, cur_load:%d, cur_idx:%d", remote_info_.c_str(), load, next_load_idx_);
- }
- ++next_load_idx_;
-
- return true;
- }
-
- uint32_t Connection::parseAttr(char *attr, int32_t attr_len)
- {
- char *mid = nullptr;
- LOG_TRACE("ack attr:%s", attr);
-
- mid = strstr(attr, "mid=");
- if (!mid)
- {
- if (attr[attr_len - 1] != '\0')
- {
- attr[attr_len - 1] = '\0';
- LOG_ERROR("force show len(%d) attr:%s.", attr_len, attr);
- }
- else
- {
- LOG_ERROR("show len(%d) attr:%s.", attr_len, attr);
- }
-
- return -1;
- }
-
- uint32_t buf_uniqId = atoi(&mid[4]);
- LOG_TRACE("parse ack and get buf uid:%d", buf_uniqId);
-
- return buf_uniqId;
- }
-
- int Connection::parseErrorCode(char *attr, int32_t attr_len) {
- char *errorCodeStr = nullptr;
- LOG_TRACE("ack attr:%s", attr);
- errorCodeStr = strstr(attr, "errCode=");
- if (!errorCodeStr) {
- return 0;
- }
- int errorCode = atoi(&errorCodeStr[8]);
- LOG_TRACE("parse ack and get buf errorCode:%d", errorCode);
-
- return errorCode;
- }
-
- int32_t Connection::getAvgLoad()
- {
- if (isStop())
- return -1;
-
- std::lock_guard<std::mutex> lck(load_mutex_);
- int32_t numerator = 0;
- int32_t denominator = 0;
- for (int i = 0; i < loads_.size(); i++)
- {
- if (loads_[i] > 0)
- {
- numerator += loads_[i] * constants::kWeight[i];
- denominator += constants::kWeight[i];
- }
- }
- if (0 == denominator)
- return 0;
- return numerator / denominator;
- }
-
- void Connection::setLocalInfo()
- {
- local_info_ = "[ip:" + socket_->local_endpoint().address().to_string() + ", port:" + std::to_string(socket_->local_endpoint().port()) + "]";
- }
-
-} // namespace dataproxy_sdk
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/socket_connection.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/socket_connection.h
deleted file mode 100644
index 9b2cbe0534..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/net/socket_connection.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef DATAPROXY_SDK_NET_CONNECTION_H_
-#define DATAPROXY_SDK_NET_CONNECTION_H_
-
-#include <asio.hpp>
-#include <deque>
-#include <functional>
-#include <memory>
-#include <mutex>
-#include <system_error>
-#include <unordered_map>
-
-#include "sdk_core.h"
-#include "executor_thread_pool.h"
-#include "logger.h"
-#include "noncopyable.h"
-#include "recv_buffer.h"
-#include "msg_protocol.h"
-
-namespace dataproxy_sdk
-{
- class SendBuffer;
- class Connection : noncopyable, public std::enable_shared_from_this<Connection>
- {
- public:
- enum Status
- {
- kConnecting,
- kConnected,
- kDisconnected
- };
-
- private:
- ExecutorThreadPtr executor_; //network sending thread, executor:socket=1:n
- TcpSocketPtr socket_;
- std::atomic<Status> status_;
- SteadyTimerPtr timer_;
- ProxyInfoPtr proxyinfo_;
- std::string local_info_; //local ip+port
- std::string remote_info_;
-
- std::deque<SendBuffer *> write_queue_; //waiting for sending
- RecvBufferPtr recv_buf_;
-
- BinaryHB binHB_ = {0}; //binary hb
- char msgHB[5] = {0, 0, 0, 0x1, 0x1};
-
- std::vector<int32_t> loads_; // connection loads in last 30 times
- int32_t next_load_idx_;
-
- int32_t thread_id_;
-
- enum
- {
- kConnectTimeout = 1000 * 20, //ms //FIXME: improve, need fix?
- };
-
- AtomicUInt send_err_nums_;
- AtomicUInt total_send_; //total send buf
- AtomicUInt total_read_; //total read ack package
- AtomicInt waiting_send_;
- AtomicInt retry_hb_; //hb retry times, reset while receiving ack
-
- mutable std::mutex load_mutex_; // loads lock
-
- public:
- Connection(ExecutorThreadPtr &executor, ProxyInfoPtr &proxyinfo);
- ~Connection();
-
- void sendBuf(SendBuffer* buf);
- void sendHB(bool isBinHB);
- void connClose();
- Status status() const { return status_; }
- inline bool isStop() const { return status_ == kDisconnected; }
- inline bool isConnected() const { return status_ == kConnected; }
- int32_t getThreadId() const { return thread_id_; }
- inline int32_t getWaitingSend() { return waiting_send_.get(); }
- inline void decreaseWaiting()
- {
- if (waiting_send_.get() > 0)
- {
- waiting_send_.decrement();
- executor_->waiting_send_.decrement();
- }
- }
- std::string getRemoteInfo() const { return remote_info_; }
- ProxyInfoPtr getBusInfo() const { return proxyinfo_; }
-
- int32_t getAvgLoad();
-
- private:
- void doConnect(const asio::ip::tcp::endpoint &ep);
- void doWrite(); //async send data
- void doRead();
- void connectHandler(const std::error_code &ec);
- void setLocalInfo();
- void doClose(const std::error_code *err = nullptr);
- int32_t doParse();
- bool parseProtocolAck(uint32_t total_len);
- bool parseBinaryAck(uint32_t total_len);
- bool parseBinaryHB(uint32_t total_len);
- uint32_t parseAttr(char *attr, int32_t attr_len);
- int parseErrorCode(char *attr, int32_t attr_len);
- };
-
-} // namespace dataproxy_sdk
-
-#endif // DATAPROXY_SDK_NET_CONNECTION_H_
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/capi_constant.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/capi_constant.h
index e045171de3..89f10e5656 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/capi_constant.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/capi_constant.h
@@ -55,7 +55,8 @@ static const uint32_t kLogSize = 100 * 1024 * 1024;
static const uint8_t kLogLevel = 2;
static const char kLogPath[] = "./";
-static const char kBusURL[] = "127.0.0.1/api/tdbus_ip_v2";
+static const char kManagerURL[] =
+ "http://127.0.0.1:8099/inlong/manager/openapi/dataproxy/getIpList";
static const bool kEnableBusURLFromCluster = false;
static const char kBusClusterURL[] =
"127.0.0.1/heartbeat/tdbus_ip_v2?cluster_id=0&net_tag=all";
@@ -80,7 +81,8 @@ static const char kBasicAuthHeader[] = "Authorization:";
static const char kBasicAuthPrefix[] = "Basic";
static const char kBasicAuthSeparator[] = " ";
static const char kBasicAuthJoiner[] = ":";
-static const char kProtocolType [] = "TCP";
+static const char kProtocolType[] = "TCP";
+static const bool kNeedAuth = false;
static const uint32_t kMaxAttrLen = 2048;
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/utils.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/utils.cc
index f4810e92f3..5fd060a07a 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/utils.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/utils.cc
@@ -41,7 +41,8 @@
#include <sys/time.h>
#include "api_code.h"
-#include "tc_api.h"
+#include "capi_constant.h"
+
namespace inlong {
uint16_t Utils::sequence = 0;
uint64_t Utils::last_msstamp = 0;
@@ -49,6 +50,13 @@ char Utils::snowflake_id[35] = {0};
AtomicUInt g_send_msgid{0};
AtomicInt user_exit_flag{0};
+char Base64Table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
+ 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+ 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+ 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
+ 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2',
+ '3', '4', '5', '6', '7', '8', '9', '+', '/'};
+
void Utils::taskWaitTime(int32_t sec) {
struct timeval tv;
tv.tv_sec = sec;
@@ -355,7 +363,122 @@ int32_t Utils::requestUrl(const std::string &url, std::string &urlByDNS,
return 0;
}
+std::string Utils::Base64Encode(const std::string &data) {
+ size_t in_len = data.size();
+ size_t out_len = 4 * ((in_len + 2) / 3);
+ std::string ret(out_len, '\0');
+ size_t i;
+ char *p = const_cast<char *>(ret.c_str());
+
+ for (i = 0; i < in_len - 2; i += 3) {
+ *p++ = Base64Table[(data[i] >> 2) & 0x3F];
+ *p++ =
+ Base64Table[((data[i] & 0x3) << 4) | ((int)(data[i + 1] & 0xF0) >> 4)];
+ *p++ = Base64Table[((data[i + 1] & 0xF) << 2) |
+ ((int)(data[i + 2] & 0xC0) >> 6)];
+ *p++ = Base64Table[data[i + 2] & 0x3F];
+ }
+ if (i < in_len) {
+ *p++ = Base64Table[(data[i] >> 2) & 0x3F];
+ if (i == (in_len - 1)) {
+ *p++ = Base64Table[((data[i] & 0x3) << 4)];
+ *p++ = '=';
+ } else {
+ *p++ = Base64Table[((data[i] & 0x3) << 4) |
+ ((int)(data[i + 1] & 0xF0) >> 4)];
+ *p++ = Base64Table[((data[i + 1] & 0xF) << 2)];
+ }
+ *p++ = '=';
+ }
+ return ret;
+}
+
+std::string Utils::GenBasicAuthCredential(const std::string &id,
+ const std::string &key) {
+ std::string credential = id + constants::kBasicAuthJoiner + key;
+ std::string result = constants::kBasicAuthPrefix;
+ result.append(constants::kBasicAuthSeparator);
+ result.append(Base64Encode(credential));
+ return result;
+}
+int32_t Utils::requestUrl(std::string &res, const HttpRequest *request) {
+ CURL *curl = NULL;
+ struct curl_slist *list = NULL;
+
+ curl_global_init(CURL_GLOBAL_ALL);
+
+ curl = curl_easy_init();
+ if (!curl) {
+ LOG_ERROR("failed to init curl object");
+ return SdkCode::kErrorCURL;
+ }
+
+ // http header
+ list = curl_slist_append(list,
+ "Content-Type: application/x-www-form-urlencoded");
+
+ if (request->need_auth && !request->auth_id.empty() &&
+ !request->auth_key.empty()) {
+ // Authorization: Basic xxxxxxxx
+ std::string auth = constants::kBasicAuthHeader;
+ auth.append(constants::kBasicAuthSeparator);
+ auth.append(GenBasicAuthCredential(request->auth_id, request->auth_key));
+ LOG_INFO("request manager, auth-header:" << auth.c_str());
+ list = curl_slist_append(list, auth.c_str());
+ }
+
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
+
+ // set url
+ curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
+ curl_easy_setopt(curl, CURLOPT_URL, request->url.c_str());
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request->post_data.c_str());
+ curl_easy_setopt(curl, CURLOPT_TIMEOUT, request->timeout);
+
+ // register callback and get res
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &Utils::getUrlResponse);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, &res);
+ curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
+
+ // execute curl request
+ CURLcode ret = curl_easy_perform(curl);
+ if (ret != 0) {
+ LOG_ERROR(curl_easy_strerror(ret));
+ LOG_ERROR("failed to request data from %s" << request->url.c_str());
+ if (curl)
+ curl_easy_cleanup(curl);
+ curl_global_cleanup();
+
+ return SdkCode::kErrorCURL;
+ }
+
+ int32_t code;
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
+ if (code != 200) {
+ LOG_ERROR("tdm responsed with code %d" << code);
+ if (curl)
+ curl_easy_cleanup(curl);
+ curl_global_cleanup();
+
+ return SdkCode::kErrorCURL;
+ }
+
+ if (res.empty()) {
+ LOG_ERROR("tdm return empty data");
+ if (curl)
+ curl_easy_cleanup(curl);
+ curl_global_cleanup();
+
+ return SdkCode::kErrorCURL;
+ }
+
+ // clean work
+ curl_easy_cleanup(curl);
+ curl_global_cleanup();
+
+ return 0;
+}
size_t Utils::getUrlResponse(void *buffer, size_t size, size_t count,
void *response) {
std::string *str = (std::string *)response;
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/utils.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/utils.h
index 4ccdfa0aef..c1d731f4dc 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/utils.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/utils/utils.h
@@ -33,6 +33,16 @@
namespace inlong {
extern AtomicUInt g_send_msgid;
extern AtomicInt user_exit_flag;
+struct HttpRequest
+{
+ std::string url;
+ uint32_t timeout;
+ bool need_auth;
+ std::string auth_id;
+ std::string auth_key;
+ std::string post_data;
+
+};
class Utils {
private:
static char snowflake_id[35];
@@ -54,6 +64,7 @@ class Utils {
static bool bindCPU(int32_t cpu_id);
static int32_t requestUrl(const std::string& url, std::string& urlByDNS, std::string& res,
uint32_t timeout);
+ static int32_t requestUrl(std::string& res, const HttpRequest* request);
static bool readFile(const std::string& file_path,
std::string& content);
@@ -61,7 +72,10 @@ class Utils {
const std::string& delimiter);
static std::string getVectorStr(std::vector<std::string>& vs);
- private:
+ static std::string GenBasicAuthCredential(const std::string& id, const std::string& key);
+ static std::string Base64Encode(const std::string& data);
+
+private:
static size_t getUrlResponse(void* buffer, size_t size, size_t count, void* response);
static int64_t waitNextMills(int64_t last_ms);
static std::string trim(const std::string& source);
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/CMakeLists.txt b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/CMakeLists.txt
deleted file mode 100644
index f543df660f..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/CMakeLists.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-function(sdk_add_tests _name)
- set(_srcs ${ARGN})
- message(STATUS "${_name} sources: ${_srcs}")
- add_executable(${_name} ${_srcs})
- include_directories(release/inc)
- target_link_libraries(${_name} sdk_base sdk_net pthread gtest gmock)
-endfunction(sdk_add_tests _name)
-
-
-add_subdirectory(base)
-add_subdirectory(net)
-add_subdirectory(main)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/CMakeLists.txt b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/CMakeLists.txt
deleted file mode 100644
index 9ea3956678..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/CMakeLists.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-sdk_add_tests(test_utils test_utils.cc)
-sdk_add_tests(client_config_test client_config_test.cc)
-sdk_add_tests(proxylist_config_test proxylist_config_test.cc)
-sdk_add_tests(test_log test_log.cc)
-sdk_add_tests(pack_queue_test pack_queue_test.cc)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/client_config_test.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/client_config_test.cc
deleted file mode 100644
index cecfbb1b51..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/client_config_test.cc
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-#define private public
-#include "atomic.h"
-#include "client_config.h"
-#include "logger.h"
-#include "tc_api.h"
-#include "utils.h"
-#include <gtest/gtest.h>
-#include <iostream>
-#include <stdint.h>
-#include <string>
-#include <thread>
-using namespace std;
-using namespace dataproxy_sdk;
-
-TEST(clientBase, test1)
-{
- ClientConfig client;
- EXPECT_EQ(client.parseConfig("config.json"), true);
- // EXPECT_EQ(client.bufNum(), 1204);
-
- EXPECT_EQ(client.parseConfig("nochconfig.json"), false);
-}
-
-TEST(client, test2)
-{
- ClientConfig client;
- EXPECT_EQ(client.parseConfig("emptyconfig.json"), true);
-}
-
-TEST(client, init)
-{
- ClientConfig client;
- client.proxy_URL_ = "proxy_url";
- client.auth_key_ = "key";
- EXPECT_EQ(client.proxy_URL_, "proxy_url");
- EXPECT_EQ(client.need_auth_, false);
- EXPECT_EQ(client.auth_id_, "");
- EXPECT_EQ(client.auth_key_, "key");
- EXPECT_EQ(client.enable_pack_, constants::kEnablePack);
-}
-
-TEST(sdk, init)
-{
- ClientConfig client;
- int32_t init_first = tc_api_init("./release/conf/config_example.json");
- int32_t init_second = tc_api_init(client);
- EXPECT_EQ(init_first, 0);
- EXPECT_EQ(init_second, SDKInvalidResult::kMultiInit);
- EXPECT_EQ(tc_api_close(1000), 0);
-
-}
-
-int main(int argc, char* argv[])
-{
- testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/pack_queue_test.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/pack_queue_test.cc
deleted file mode 100644
index 4823aa8584..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/pack_queue_test.cc
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-#define private public
-#include "../common.h"
-
-using namespace testing;
-
-// #if 0
-TEST(packqueue, basetest)
-{
- g_config.msg_type_ = 3;
- g_config.retry_num_ = 100;
- g_config.enable_pack_ = false;
-
- ExecutorThreadPtr th1 = make_shared<ExecutorThread>(1);
- ProxyInfoPtr proxy = make_shared<ProxyInfo>(1, "127.0.0.1", 4000);
- ConnectionPtr conn1 = make_shared<Connection>(th1, proxy);
-
- string inlong_group_id = "inlong_groupid_1";
- string inlong_stream_id = "inlong_streamid_1";
- PackQueuePtr q = make_shared<PackQueue>(inlong_group_id, inlong_stream_id);
- string msg = "test testwoehgorirehjtirjhhgklwpgJpwjgehreahethtn aethrtshtrs";
- cout << msg.size() << endl;
- string attr = "inlong_group_id=inlong_groupid_1&inlong_stream_id=inlong_streamid_1&dt=1637484644807&mid=1&cnt=1&sid=0x098714192b3bed27144b48f621800000";
- cout << "attrlen:" << attr.size() << endl;
-
- EXPECT_EQ(q->appendMsg(msg, "", 0, NULL), 0);
- EXPECT_NE(q->dataTime(), 0);
- EXPECT_EQ(q->curLen(), msg.size() + 1);
- cout << q->data() << endl;
- EXPECT_EQ(q->inlong_group_id(), inlong_group_id);
- string topic = "inlong_group_id=" + inlong_group_id + "&inlong_stream_id=" + inlong_stream_id;
- EXPECT_EQ(q->topicDesc(), topic);
- // EXPECT_EQ(q->isTriggerPack(0, msg.size()), false);
-
- TotalPoolsMock* gMock = new TotalPoolsMock();
- BufferPoolMockPtr poolMock = make_shared<BufferPoolMock>(0, 10, 1000);
- EXPECT_CALL(*gMock, getPool).WillRepeatedly(Return(poolMock));
- g_pools = gMock;
- // EXPECT_EQ(g_pools->getPool(0), poolMock);
-
- GlobalClusterMock* clusterMock = new GlobalClusterMock();
- EXPECT_CALL(*clusterMock, getSendConn).WillRepeatedly(Return(conn1));
- g_clusters = clusterMock;
- // EXPECT_EQ(g_clusters->getSendConn("bidd"), conn1);
-
- // EXPECT_EQ(q->writeToBuf(), 0);
-
- this_thread::sleep_for(chrono::minutes(1));
-
- delete gMock;
- delete clusterMock;
-}
-// #endif
-TEST(packzip, test)
-{
- g_config.msg_type_ = 3;
- g_config.enable_pack_ = false;
- g_config.enable_zip_ = false;
-
- string msg2(1024, 'a');
- msg2 += "end";
- cout << "msg2 len" << msg2.size() << endl;
-
- string inlong_group_id = "inlong_groupid_1";
- string inlong_stream_id = "inlong_streamid_1";
- PackQueuePtr q = make_shared<PackQueue>(inlong_group_id, inlong_stream_id);
- q->appendMsg(msg2, "", 0, NULL);
-
- SendBuffer* buf = new SendBuffer(40000000);
- uint32_t outlen = 0;
- q->packOperate(buf->content(), outlen, 10);
- buf->setLen(outlen);
-}
-
-TEST(packqueue, packtest)
-{
- g_config.msg_type_ = 5;
- g_config.retry_num_ = 100;
- g_config.enable_pack_ = false;
- g_config.enable_zip_ = false;
-
- string inlong_group_id = "inlong_groupid_1";
- string inlong_stream_id = "inlong_streamid_1";
- PackQueuePtr q = make_shared<PackQueue>(inlong_group_id, inlong_stream_id);
- string msg(1024, 'a');
- cout << msg.size() << endl;
- string attr = "inlong_group_id=inlong_groupid_1&inlong_stream_id=inlong_streamid_1&dt=1637484644807&mid=1&cnt=1&sid=0x098714192b3bed27144b48f621800000";
- cout << "attrlen:" << attr.size() << endl;
-
- q->appendMsg(msg, "", 0, NULL);
- string topic = "inlong_group_id=" + inlong_group_id + "&inlong_stream_id=" + inlong_stream_id;
-
- SendBuffer* buf = new SendBuffer(20000);
-
- if (!buf) return;
- uint32_t out_len = 0;
- LOG_DEBUG("buf content is%d", buf->content());
- q->packOperate(buf->content(), out_len, 1);
- buf->setLen(out_len);
- LOG_DEBUG("buf content is%d", buf->content());
- delete buf;
-}
-
-int main(int argc, char* argv[])
-{
-
- g_config.parseConfig("config.json");
- Utils::getFirstIpAddr(g_config.ser_ip_);
-
- testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/proxylist_config_test.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/proxylist_config_test.cc
deleted file mode 100644
index 41d437669e..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/proxylist_config_test.cc
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-#define private public
-#include "../common.h"
-
-TEST(businfo, basetest)
-{
- ProxyInfoPtr proxy = make_shared<ProxyInfo>(1, "0.0.0.0", 11009);
- EXPECT_EQ(proxy->getString(), "[ip:0.0.0.0, port:11009]");
-}
-
-TEST(clusterProxylist, readCachelist) //.bus_list.ini文件中缓存有groupid_test1和groupid_test2
-{
- EXPECT_EQ(g_config.parseConfig("config.json"), true);
- EXPECT_EQ(Utils::getFirstIpAddr(g_config.ser_ip_), true);
-
- GlobalClusterMock *m_clusters = new GlobalClusterMock();
-
- g_clusters = m_clusters;
- EXPECT_CALL(*m_clusters, printAckNum).Times(1);
-
- g_executors = new ExecutorThreadPool();
-
- this_thread::sleep_for(std::chrono::seconds(5));
- g_clusters->startUpdateSubroutine();
-
- g_clusters->addBuslist("test_inlong_group_id1");
- g_clusters->addBuslist("groupid_test1");
- g_clusters->addBuslist("groupid_test2");
-
- this_thread::sleep_for(std::chrono::minutes(10));
-
- delete g_clusters;
- delete g_executors;
-}
-
-TEST(clusterProxylist, basetest)
-{
- EXPECT_EQ(g_config.parseConfig("config.json"), true);
- ClusterProxyListPtr cluster = make_shared<ClusterProxyList>();
- GlobalCluster *g_cluster = new GlobalCluster();
- string meta_info;
- EXPECT_EQ(Utils::readFile("proxylist_old.json", meta_info), true);
- // LOG_INFO("%s", meta_info.c_str());
-
- EXPECT_EQ(g_cluster->parseAndGet("test_group", meta_info, cluster), 0);
- EXPECT_EQ(cluster->size(), 53);
- EXPECT_EQ(cluster->clusterId(), 70);
- EXPECT_EQ(cluster->switchValue(), 1);
- EXPECT_EQ(cluster->isNeedLoadBalance(), true);
-
- string info;
- ClusterProxyListPtr cluster2 = make_shared<ClusterProxyList>();
- EXPECT_EQ(Utils::readFile("proxy_new.json", info), true);
- EXPECT_EQ(g_cluster->parseAndGet("test_group", info, cluster2), 0);
- EXPECT_EQ(cluster2->size(), 2);
- EXPECT_EQ(cluster2->clusterId(), 45);
- EXPECT_EQ(cluster2->switchValue(), 0);
- EXPECT_EQ(cluster2->load(), 20);
- EXPECT_EQ(cluster2->isNeedLoadBalance(), false);
- delete g_cluster;
-}
-
-TEST(clusterProxylist, addBuslistAndUpdate1)
-{
- EXPECT_EQ(g_config.parseConfig("config.json"), true);
- g_clusters = new GlobalCluster();
- g_executors = new ExecutorThreadPool();
- this_thread::sleep_for(std::chrono::seconds(5));
-
- g_clusters->startUpdateSubroutine();
-
- this_thread::sleep_for(std::chrono::seconds(2));
-
- g_clusters->addBuslist("test_inlong_group_id1");
-
- this_thread::sleep_for(std::chrono::minutes(10));
-
- delete g_clusters;
- delete g_executors;
- // this_thread::sleep_for(std::chrono::seconds(10));
-}
-
-void addWrongBidFunc(GlobalCluster *gcluster)
-{
- // this_thread::sleep_for(std::chrono::seconds(30));
-
- gcluster->addBuslist("b_wrong_bid");
- gcluster->addBuslist("b_ssssssssid");
-}
-
-void addExistBidFunc(GlobalCluster *gcluster)
-{
- // this_thread::sleep_for(std::chrono::seconds(15));
-
- gcluster->addBuslist("test_inlong_group_id1");
-}
-
-TEST(clusterProxylist, connInit)
-{
- EXPECT_EQ(g_config.parseConfig("config.json"), true);
- EXPECT_EQ(Utils::getFirstIpAddr(g_config.ser_ip_), true);
-
- g_clusters = new GlobalCluster();
- g_executors = new ExecutorThreadPool();
-
- this_thread::sleep_for(std::chrono::seconds(5));
- g_clusters->startUpdateSubroutine();
-
- g_clusters->addBuslist("test_inlong_group_id1");
-
- this_thread::sleep_for(std::chrono::seconds(1));
-
- std::thread th1(bind(addWrongBidFunc, g_clusters));
- th1.detach();
- std::thread th2(bind(addExistBidFunc, g_clusters));
- th2.detach();
-
- this_thread::sleep_for(std::chrono::minutes(20));
-
- delete g_clusters;
- delete g_executors;
-
- this_thread::sleep_for(std::chrono::minutes(1));
-}
-
-int main(int argc, char *argv[])
-{
- testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/test_log.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/test_log.cc
deleted file mode 100644
index 5b207f54de..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/test_log.cc
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "atomic.h"
-#include "logger.h"
-#include "singleton.h"
-#include <string>
-#include <thread>
-using namespace dataproxy_sdk;
-using namespace std;
-
-AtomicInt ati;
-
-void logfunc()
-{
- // for (int i = 0; i < 100; i++) {
- while (true)
- {
- LOG_INFO("%d, this is info log %d", ati.incrementAndGet(), 1);
- std::string str = "tttttt";
- LOG_TRACE("%d, this is trace log:%s", ati.incrementAndGet(), str.c_str());
- LOG_ERROR("%d, this is an error log", ati.incrementAndGet());
- LOG_WARN("%d, warn log: "
- "wartryjreah5p3ihj504twujyhi6twjuhyit6jioawjhyiow4ehjygio3aqjhyginareowigpoijewpto4urgt94ure9guer9itug",
- ati.incrementAndGet());
- LOG_DEBUG("%d, debug llllllllllllll %d", ati.incrementAndGet(), 124);
- }
-}
-
-int main(int argc, char const* argv[])
-{
- debug_init_log();
- LOG_INFO("this is info log %d", 1);
- std::string str = "tttttt";
- LOG_TRACE("this is trace log:%s", str.c_str());
- LOG_ERROR("this is an error log");
- LOG_WARN("warn log: warnareowigpoijewpto4urgt94ure9guer9itug");
- LOG_DEBUG("debug llllllllllllll %d", 124);
-
- ati.getAndSet(0);
- std::thread t1(logfunc);
- std::thread t2(logfunc);
- std::thread t3(logfunc);
- std::thread t4(logfunc);
-
- t1.join();
-
- return 0;
-}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/test_utils.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/test_utils.cc
deleted file mode 100644
index 33e23de283..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/base/test_utils.cc
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-#define private public
-
-#include "atomic.h"
-#include "logger.h"
-#include "tc_api.h"
-#include "utils.h"
-#include <algorithm>
-#include <gtest/gtest.h>
-#include <iostream>
-#include <map>
-#include <stdint.h>
-#include <string>
-#include <thread>
-using namespace std;
-using namespace dataproxy_sdk;
-
-AtomicInt init_flag;
-
-TEST(sort, test)
-{
- map<string, int> result;
- result["wegre"] = 3;
- result["aaaaa"] = 1;
- result["bbbbbbb"] = 0;
- result["test"] = 1;
- result["ggggg"] = 9;
- result["ddddd"] = -1;
- vector<PAIR> vlist(result.begin(), result.end());
- sort(vlist.begin(), vlist.end(), &Utils::upValueSort);
- cout << "up sort" << endl;
- for (size_t i = 0; i < vlist.size(); i++)
- {
- cout << vlist[i].first << ":" << vlist[i].second << endl;
- }
-
- cout << "down sort" << endl;
- sort(vlist.begin(), vlist.end(), &Utils::downValueSort);
- for (size_t i = 0; i < vlist.size(); i++)
- {
- cout << vlist[i].first << ":" << vlist[i].second << endl;
- }
-}
-
-TEST(timetest, test)
-{
- uint64_t t1 = Utils::getCurrentMsTime();
- Utils::taskWaitTime(1);
- uint64_t t2 = Utils::getCurrentMsTime();
- ASSERT_EQ(t1 + 1000, t2);
-}
-
-TEST(bindCPU, test) { ASSERT_EQ(Utils::bindCPU(10), true); }
-
-TEST(getIpAddress, test)
-{
- string ip;
- ASSERT_EQ(Utils::getFirstIpAddr(ip), true);
- // ASSERT_NE(ip, "11.45.17.175");
- ASSERT_NE(ip, "127.0.0.1");
-}
-
-TEST(getFormatCurTime, test)
-{
- string curtime = Utils::getFormatTime(Utils::getCurrentMsTime());
- cout << curtime << endl;
-}
-
-TEST(waitNextMills, test)
-{
- uint64_t t1 = Utils::getCurrentMsTime();
- Utils::waitNextMills(t1);
- uint64_t t2 = Utils::getCurrentMsTime();
- ASSERT_EQ(t1 + 1, t2);
-}
-
-TEST(isLegalTime, test)
-{
- ASSERT_NE(Utils::isLegalTime(0), true);
- ASSERT_EQ(Utils::isLegalTime(2435101567000), true);
-}
-
-TEST(snowflake, test)
-{
- string id1 = Utils::getSnowflakeId();
- Utils::taskWaitTime(3);
- string id2 = Utils::getSnowflakeId();
- ASSERT_NE(id1, id2);
-}
-
-TEST(requestUrl, test)
-{
- string str;
- string url = "http://127.0.0.1:8099/api/inlong/manager/openapi/dataproxy/getIpList?inlong_group_id=test_group_id";
- HttpRequest request = {url, 1000, false, "", "", str};
- int32_t res = Utils::requestUrl(url, &request);
- cout << str << endl;
- ASSERT_EQ(res, 0);
-}
-
-TEST(zip, test)
-{
- string input = "Running transaction check---> Package glibc-debuginfo.i686 0:2.12-1.132.el6 will be installed--> Processing"
- "Dependency: glibc-debuginfo-common = 2.12-1.132.el6 for package: glibc-debuginfo-2.12-1.132.el6.i686---> Package "
- "nss-softokn-debuginfo.i686 0:3.12.9-11.el6 will be installe---> Package yum-plugin-auto-update-debug-info.noarch "
- "0:1.1.30-17.el6_5 will be installe--> Running transaction chec---> Package glibc-debuginfo-common.i686 "
- "0:2.12-1.132.el6buginfo-2.12-1.132.el6.i686---> Packagebuginfo-2.12-1.132.el6.i686---> "
- "Packagebuginfo-2.12-1.132.el6.i686---> Package "
- "will be installed-- > Finished Dependency Resolution ";
- // string input = "827206790347609547869045780928509482-2058409869054869052";
-
- string res;
- Utils::zipData(input.c_str(), input.size(), res);
- cout << "zip len: " << input.size() << ", after zip: " << res.size() << endl;
- cout << "zip res: " << res << endl;
- ASSERT_LE(res.size(), input.size());
-}
-
-TEST(zip, test2)
-{
- string input(1024, 'a');
- input += "end\n";
-
- string output;
-
- Utils::zipData(input.data(), input.size(), output);
-
- cout << "zip,len:" << output.size() << endl;
- cout << "output:" << output << endl;
-
- string res;
- snappy::Uncompress(output.data(), output.size(), &res);
- cout << "uncompress,len:" << res.size() << endl;
- cout << "uncompress content:" << res << endl;
-
- EXPECT_EQ(res, input);
-}
-
-TEST(atomic, test)
-{
- int res = 0;
- if (!init_flag.compareAndSwap(0, 1)) { res = SDKInvalidResult::kMultiInit; }
- ASSERT_EQ(res, 0);
- if (!init_flag.compareAndSwap(0, 1)) { res = SDKInvalidResult::kMultiInit; }
- ASSERT_EQ(res, 4);
-}
-
-TEST(atomic, test2)
-{
- int res = 0;
- if (!init_flag.compareAndSwap(0, 1)) { res = SDKInvalidResult::kMultiInit; }
- ASSERT_EQ(res, 4);
-}
-
-TEST(readfile, test1)
-{
- string file_path = "config.json";
- string wrong_path = "ddddd.txt";
- string res;
- EXPECT_EQ(Utils::readFile(file_path, res), true);
- cout << "res: " << res << endl;
- EXPECT_EQ(Utils::readFile(wrong_path, res), false);
-}
-
-TEST(splitStr, test)
-{
- string a = "ewigjreiher,oeirhgoer,oeaihrn__gee,jj";
- string b = "big,bibbb,";
- string c = " ";
- string d = "b_bidlone";
- string e = " , ";
- string f = "big,bibbb, ";
- vector<string> av, bv, cv, dv, ev, fv;
- EXPECT_EQ(Utils::splitOperate(a, av, ","), 4);
- EXPECT_EQ(Utils::splitOperate(b, bv, ","), 2);
- EXPECT_EQ(Utils::splitOperate(c, cv, ","), 0);
- EXPECT_EQ(Utils::splitOperate(d, dv, ","), 1);
- EXPECT_EQ(Utils::splitOperate(e, ev, ","), 0);
- EXPECT_EQ(Utils::splitOperate(f, fv, ","), 2);
- cout << Utils::getVectorStr(av) << endl;
-}
-
-int main(int argc, char* argv[])
-{
- testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/common.h b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/common.h
deleted file mode 100644
index 5d9bef4a52..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/common.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "atomic.h"
-#include "buffer_pool.h"
-#include "proxylist_config.h"
-#include "sdk_core.h"
-#include "executor_thread_pool.h"
-#include "logger.h"
-#include "msg_protocol.h"
-#include "pack_queue.h"
-#include "send_buffer.h"
-#include "socket_connection.h"
-#include "tc_api.h"
-#include "utils.h"
-#include <algorithm>
-#include <asio.hpp>
-#include <chrono>
-#include <functional>
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
-#include <iostream>
-#include <map>
-#include <memory>
-#include <stdint.h>
-#include <string>
-#include <thread>
-#include <utility>
-using namespace std;
-using namespace dataproxy_sdk;
-using namespace testing;
-
-class BufferPoolMock : public BufferPool
-{
- public:
- BufferPoolMock(int32_t id, uint32_t buf_num, uint32_t buf_size) : BufferPool(id, buf_num, buf_size) {}
-};
-using BufferPoolMockPtr = shared_ptr<BufferPoolMock>;
-
-class TotalPoolsMock : public TotalPools
-{
- public:
- TotalPoolsMock() : TotalPools() {}
- MOCK_METHOD1(getPool, BufferPoolPtr(int32_t));
-};
-
-class GlobalQueuesMock : public GlobalQueues
-{
- GlobalQueuesMock() : GlobalQueues() {}
- MOCK_METHOD2(getPackQueue, PackQueuePtr(const std::string&, const std::string&));
-};
-
-class ClusterProxyListMock : public ClusterProxyList
-{
- ClusterProxyListMock() : ClusterProxyList() {}
- MOCK_METHOD0(isNeedLoadBalance, bool(void));
- MOCK_METHOD0(initConn, int32_t(void));
-};
-using ClusterBuslistMockPtr = shared_ptr<ClusterProxyListMock>;
-
-class GlobalClusterMock : public GlobalCluster
-{
- public:
- GlobalClusterMock() {}
- MOCK_METHOD1(getSendConn, ConnectionPtr(const std::string&));
- MOCK_METHOD1(printAckNum, void(const std::error_code&));
-};
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/CMakeLists.txt b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/CMakeLists.txt
deleted file mode 100644
index 534d9a89cc..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/CMakeLists.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-sdk_add_tests(base_core_test base_core_test.cc)
-sdk_add_tests(init_exit_test init_exit_test.cc)
-sdk_add_tests(multi_thread_send multi_thread_send.cc)
-sdk_add_tests(random_send random_send.cc)
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/base_core_test.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/base_core_test.cc
deleted file mode 100644
index a2c8da3e71..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/base_core_test.cc
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "../common.h"
-#include "tc_api.h"
-#include <chrono>
-#include <iostream>
-#include <string>
-#include <thread>
-
-using namespace std;
-using namespace dataproxy_sdk;
-
-int run_times = 1;
-
-TEST(msgsize, size100Btest)
-{
- string msg2(100, 'a');
- msg2 += "end";
- for (int32_t i = 0; i < run_times; i++)
- {
- int32_t ret2 = tc_api_send("inlong_groupid_test", "100Btest", msg2.c_str(), msg2.length());
- cout << ret2 << "; ";
- }
-}
-
-TEST(msgsize, size1Ktest)
-{
- string msg2(1024, 'a');
- msg2 += "end";
- for (int32_t i = 0; i < run_times; i++)
- {
- int32_t ret2 = tc_api_send("inlong_groupid_test", "1Ktest", msg2.c_str(), msg2.length());
- cout << ret2 << "; ";
- }
-}
-
-TEST(msgsize, size100Ktest)
-{
- string msg2(1024 * 100, 'a');
- msg2 += "end";
- for (int32_t i = 0; i < run_times; i++)
- {
- int32_t ret2 = tc_api_send("inlong_groupid_test", "100Ktest", msg2.c_str(), msg2.length());
- // this_thread::sleep_for(chrono::milliseconds(1));
- cout << ret2 << "; ";
- }
-}
-
-TEST(msgsize, size500Ktest)
-{
- string msg2(1024 * 500, 'a');
- msg2 += "end";
- for (int32_t i = 0; i < run_times; i++)
- {
- int32_t ret2 = tc_api_send("inlong_groupid_test", "500Ktest", msg2.c_str(), msg2.length());
- cout << ret2 << "; ";
- // this_thread::sleep_for(chrono::milliseconds(1));
- }
-}
-
-TEST(msgsize, size1Mtest)
-{
- string msg2(1024 * 1024, 'a');
- msg2 += "end";
- for (int32_t i = 0; i < run_times; i++)
- {
- int32_t ret2 = tc_api_send("inlong_groupid_test", "1Mtest", msg2.c_str(), msg2.length());
- cout << ret2 << "; ";
- }
-}
-
-TEST(msgsize, size2Mtest)
-{
- string msg2(1024 * 1024 * 2, 'a');
- msg2 += "end";
- for (int32_t i = 0; i < run_times; i++)
- {
- int32_t ret2 = tc_api_send("inlong_groupid_test", "2Mtest", msg2.c_str(), msg2.length());
- cout << ret2 << "; ";
- }
-}
-
-TEST(msgsize, size5Mtest)
-{
- string msg5(1024 * 1024 * 5, 'a');
- msg5 += "end";
- for (int i = 0; i < run_times; i++)
- {
- int32_t ret5 = tc_api_send("inlong_groupid_test", "5Mtest", msg5.c_str(), msg5.length());
- cout << ret5 << "; ";
- }
-}
-
-TEST(msgsize, size10Mtest)
-{
- string msg10(1024 * 1024 * 10, 'a');
- msg10 += "end";
- for (int i = 0; i < run_times; i++)
- {
- int32_t ret10 = tc_api_send("inlong_groupid_test", "10Mtest", msg10.c_str(), msg10.length());
- cout << ret10 << "; ";
- // this_thread::sleep_for(chrono::milliseconds(10));
- }
-}
-
-int32_t callBackFunc(string inlong_group_id, string inlong_stream_id)
-{
- cout << "inlong_group_id----" << inlong_group_id << "----inlong_stream_id" << inlong_stream_id << "send error" << endl;
- return -1;
-}
-
-int main(int argc, char* argv[])
-{
- if (tc_api_init("config.json"))
- {
- cout << "init error" << endl;
- return 0;
- }
- // this_thread::sleep_for(chrono::milliseconds(2));
- run_times = atoi(argv[1]);
- InitGoogleTest(&argc, argv);
-
- RUN_ALL_TESTS();
-
- tc_api_close(10000);
- return 0;
-}
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/init_exit_test.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/init_exit_test.cc
deleted file mode 100644
index 8a74cf4937..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/init_exit_test.cc
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "../common.h"
-
-void subroutine()
-{
- ClientConfig client_config;
- tc_api_init(client_config);
- for (size_t i = 0; i < 1000; i++)
- {
- LOG_ERROR("log error %d", i);
- LOG_STAT("log stat %d", i);
- LOG_INFO("log info %d", i);
- }
- tc_api_close(100);
-}
-
-TEST(initAndclose, test)
-{
- EXPECT_EQ(tc_api_init("config.json"), 0);
- EXPECT_EQ(tc_api_init("config.json"), SDKInvalidResult::kMultiInit);
-
- EXPECT_EQ(tc_api_close(100), 0);
- EXPECT_EQ(tc_api_close(100), SDKInvalidResult::kMultiExits);
-}
-
-TEST(multiThreadInit, test)
-{
- thread threads[20];
- for (size_t i = 0; i < 20; i++)
- {
- threads[i] = thread(subroutine);
- }
- for (size_t i = 0; i < 20; i++)
- {
- threads[i].join();
- }
-
-}
-
-int main(int argc, char* argv[])
-{
- testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/multi_thread_send.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/multi_thread_send.cc
deleted file mode 100644
index 4edf70d110..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/multi_thread_send.cc
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "../common.h"
-#include "tc_api.h"
-#include <chrono>
-#include <iostream>
-#include <stdlib.h>
-#include <string>
-#include <thread>
-#include <time.h>
-
-using namespace std;
-using namespace dataproxy_sdk;
-
-int send_times = 1;
-
-// 0:100B, 1:1K, 2:100K, 3:500K, 4:2M, 5:5M,6:10M
-uint32_t msgsize[7] = {100, 1024, 1024 * 100, 1024 * 500, 1024 * 1024 * 2, 1024 * 1024 * 5, 1024 * 1024 * 10};
-
-string base_Streamid = "cpptest_";
-string base_bid = "inlong_groupid_test";
-int Streamid_num = 10;
-int bid_num = 1;
-int milliseconds = 0;
-
-void sendFunc(const string& msg)
-{
- for (size_t i = 0; i < send_times; i++)
- {
- srand((unsigned)time(NULL));
- string inlong_stream_id = base_Streamid + to_string(rand() % Streamid_num);
-
- srand((unsigned)time(NULL));
- string inlong_group_id = base_bid + to_string(rand() % bid_num);
- int32_t ret = tc_api_send(inlong_group_id.data(), inlong_stream_id.data(), msg.data(), msg.size());
- if (ret) this_thread::sleep_for(chrono::milliseconds(milliseconds));
- }
-}
-
-void sendLimitTimes(const string& msg)
-{
- for (size_t i = 0; i < send_times; i++)
- {
- // if (i % 10 == 5) { this_thread::sleep_for(chrono::seconds(40)); } //test normal hb
-
- srand((unsigned)time(NULL));
- string inlong_stream_id = base_Streamid + to_string(rand() % Streamid_num);
-
- srand((unsigned)time(NULL));
- string inlong_group_id = base_bid + to_string(rand() % bid_num);
- int32_t ret = tc_api_send(inlong_group_id.data(), inlong_stream_id.data(), msg.data(), msg.size());
- }
-
- this_thread::sleep_for(chrono::seconds(100));
-}
-
-int main(int argc, char* argv[]) //input： thread_num, msgsize_idx, send_times
-{
- if (argc < 5)
- {
- cout << "usage: ./multi_thread_send <thread_num> <msgsize_idx> <Streamid_num> <bid_num> <milliseconds> <rumtimes>" << endl;
- return 0;
- }
-
- int thread_num = atoi(argv[1]); //send thread num
- int idx = atoi(argv[2]);
- if (idx > 6) idx = 1; //default: 10K
- uint32_t msglen = msgsize[idx];
- Streamid_num = atol(argv[3]);
- bid_num = atol(argv[4]);
-
- if (argc >= 6) milliseconds = atol(argv[5]);
- if (argc >= 7) send_times = atol(argv[6]);
-
- string msg(msglen, 'c');
- msg += "end";
-
- // #if 0
-
- if (tc_api_init("config.json"))
- {
- cout << "init error" << endl;
- return 0;
- }
-
- this_thread::sleep_for(chrono::seconds(1));
- thread t_set[thread_num];
-
- for (size_t i = 0; i < thread_num; i++)
- {
- if (send_times != 1) { t_set[i] = thread(bind(sendLimitTimes, msg)); }
- else
- {
- t_set[i] = thread(bind(sendFunc, msg));
- }
- // t.join();
- }
- for (size_t i = 0; i < thread_num; i++)
- {
- t_set[i].join();
- }
-
- tc_api_close(3000);
- // #endif
- return 0;
-}
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/random_send.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/random_send.cc
deleted file mode 100644
index 654eb1ebec..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/main/random_send.cc
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "../common.h"
-#include "tc_api.h"
-#include <chrono>
-#include <iostream>
-#include <stdlib.h>
-#include <string>
-#include <thread>
-#include <time.h>
-
-using namespace std;
-using namespace dataproxy_sdk;
-
-string base_Streamid = "cpptest_";
-string base_bid = "inlong_groupid_test";
-int milliseconds = 0;
-
-string strRand(int length)
-{
- char tmp;
- string buffer;
-
- random_device rd;
- default_random_engine random(rd());
- for (int i = 0; i < length; i++)
- {
- tmp = random() % 36;
- if (tmp < 10)
- {
- tmp += '0';
- }
- else
- {
- tmp -= 10;
- tmp += 'A';
- }
- buffer += tmp;
- }
- return buffer;
-}
-
-void sendFunc()
-{
- while (true)
- {
- srand((unsigned)time(NULL));
- string inlong_stream_id = base_Streamid + to_string(rand() % 10);
-
- srand((unsigned)time(NULL));
- string inlong_group_id = base_bid + to_string(rand() % 3);
-
- srand((unsigned)time(NULL));
- int32_t base = 1024 * 1024 * 2;
- int32_t msglen = 100 + rand() % base;
- string msg = strRand(msglen);
-
- int32_t ret = tc_api_send(inlong_group_id.data(), inlong_stream_id.data(), msg.data(), msg.size());
- if (ret) this_thread::sleep_for(chrono::milliseconds(milliseconds));
- }
-}
-
-int main(int argc, char* argv[]) //input: thread_num, msgsize_idx, send_times
-{
- if (argc < 3)
- {
- cout << "usage: ./random_send.cc <thread_num> <milliseconds>" << endl;
- return 0;
- }
-
- int thread_num = atoi(argv[1]); //send thread num
- milliseconds = atol(argv[2]);
-
- // #if 0
-
- if (tc_api_init("config.json"))
- {
- cout << "init error" << endl;
- return 0;
- }
-
- this_thread::sleep_for(chrono::seconds(1));
- thread t_set[thread_num];
-
- for (size_t i = 0; i < thread_num; i++)
- {
- t_set[i] = thread(sendFunc);
- }
- for (size_t i = 0; i < thread_num; i++)
- {
- t_set[i].join();
- }
-
- tc_api_close(10000);
- // #endif
- return 0;
-}
\ No newline at end of file
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/net/CMakeLists.txt b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/net/CMakeLists.txt
deleted file mode 100644
index 7e17761d67..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/net/CMakeLists.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-sdk_add_tests(executor_test executor_test.cc)
-sdk_add_tests(send_buffer_test send_buffer_test.cc)
-sdk_add_tests(buffer_pool_test buffer_pool_test.cc)
-sdk_add_tests(socket_connection_test socket_connection_test.cc)
-sdk_add_tests(big_pack_test big_pack_test.cc)
-
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/net/big_pack_test.cc b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/net/big_pack_test.cc
deleted file mode 100644
index 714fb7091c..0000000000
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/test/net/big_pack_test.cc
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <asio.hpp>
-#include <string>
-#include <system_error>
-
-#include "socket_connection.h"
-
-using namespace std;
-using namespace dataproxy_sdk;
-
-class BigSizeConn : public enable_shared_from_this<BigSizeConn>
-{
- private:
- ExecutorThreadPtr executor_;
- string ip_;
- uint16_t port_;
- TcpSocketPtr socket_;
- bool isConnect_;
- char* data_;
- uint32_t len_;
- uint32_t runtimes_;
- uint32_t count_;
- RecvBufferPtr recv_buf_;
-
- public:
- BigSizeConn(ExecutorThreadPtr& executor, string ip, uint16_t port)
- : executor_(executor)
- , ip_(ip)
- , port_(port)
- , socket_(std::move(executor_->createTcpSocket()))
- , isConnect_(false)
- , data_(nullptr)
- , len_(0)
- , runtimes_(10)
- , count_(0)
-
- {
- recv_buf_ = std::make_shared<RecvBuffer>();
- asio::ip::tcp::endpoint ep(asio::ip::address::from_string(ip_), port_);
- doConnect(ep);
- }
-
- void doRead()
- {
- if (!isConnect_) return;
- recv_buf_->EnsureWritableBytes(1024);
- auto self = shared_from_this();
- socket_->async_receive(asio::buffer(recv_buf_->WriteBegin(), recv_buf_->WritableBytes()),
- [self, this](const std::error_code& ec, std::size_t len) {
- if (ec)
- {
- LOG_ERROR("async read data error:%s", ec.message().c_str());
- doClose(&ec);
- return;
- }
- if (len == 0)
- {
- LOG_ERROR("async read 0 bytes");
- doClose(&ec);
- return;
- }
- recv_buf_->WriteBytes(len);
-
- //读取剩余未读完的数据，如果有的话
- std::error_code tmp_error;
- size_t left_size = socket_->available(tmp_error);
- if (left_size > 0 && !tmp_error)
- {
- recv_buf_->EnsureWritableBytes(left_size);
- size_t rlen = socket_->receive(asio::buffer(recv_buf_->WriteBegin(), left_size));
- if (rlen > 0) { recv_buf_->WriteBytes(rlen); }
- }
- LOG_DEBUG("conn read %d bytes", recv_buf_->length());
- doRead();
- });
- }
-
- void doConnect(const asio::ip::tcp::endpoint& ep)
- {
- socket_->async_connect(ep, [this](const std::error_code& ec) {
- if (ec)
- {
- isConnect_ = false;
- LOG_ERROR("async connetc error");
- doClose(&ec);
- return;
- }
- isConnect_ = true;
- doRead();
- });
- }
-
- void doClose(const std::error_code* err)
- {
- if (!isConnect_) return;
- LOG_WARN("close connnection");
- socket_->close();
- }
-
- void doWrite()
- {
- if (!isConnect_ || runtimes_ == 0) return;
- auto self = shared_from_this();
- count_++;
- LOG_DEBUG("send count:%d", count_);
- asio::async_write(*socket_, asio::buffer(data_, len_), [self, this](const std::error_code& ec, std::size_t writelen) {
- LOG_WARN("doWrite callback, writelen:%d", writelen);
- if (writelen > 0 && writelen < len_)
- {
- while (writelen < len_)
- {
- LOG_WARN("cache data(len:%d) need to be resend", len_ - writelen);
- writelen += asio::write(*socket_, asio::buffer(data_ + writelen, len_ - writelen));
- }
- }
-
- if (!ec)
- {
- if (writelen != len_) { LOG_ERROR("async write error!!!!!!!!!!!!!!!!!!!!!"); }
- --runtimes_;
-
- doWrite();
- }
- else
- {
- LOG_ERROR("write error:%s", ec.message().c_str());
- doClose(&ec);
- }
- });
- }
-
- void writeData(char* data, uint32_t len, uint32_t runtimes)
- {
- auto self = shared_from_this();
- data_ = data;
- len_ = len;
- runtimes_ = runtimes;
- executor_->postTask([self, this]() {
- LOG_WARN("-----start send------");
- if (isConnect_) { doWrite(); }
- });
- }
-};
-using BigSizeConnPtr = shared_ptr<BigSizeConn>;
-
-const uint32_t capacity = 1024 * 1204 * 11;
-char data[capacity] = {0};
-
-int main(int argc, char* argv[])
-{
- // g_config = new ClientConfig("config.json");
-
- int runtimes = 10;
- if (argc == 2) { runtimes = atoi(argv[1]); }
-
- ExecutorThreadPtr th1 = make_shared<ExecutorThread>(1);
- BigSizeConnPtr conn = make_shared<BigSizeConn>(th1, "127.0.0.1", 46803);
-
- string msg(1024 * 1024 * 10, 'a');
- msg += "end";
- string attr =
- "inlong_group_id=test_group_id&inlong_stream_id=bigpacktest&dt=1638241486778&mid=10&cnt=1&sid=0x098714199eefe7f714569014e7000000";
-
- uint32_t body_len = msg.size();
- uint32_t attr_len = attr.size();
- uint32_t total_len = 9 + body_len + attr_len;
- char msg_type = 3;
-
- char* p = data;
-
- *(uint32_t*)p = htonl(total_len);
- p += 4;
- *p = msg_type;
- ++p;
- *(uint32_t*)p = htonl(body_len);
- p += 4;
- memcpy(p, msg.data(), body_len);
- p += body_len;
- *(uint32_t*)p = htonl(attr_len);
- p += 4;
... 422 lines suppressed ...