You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@brpc.apache.org by "liuxuting (via GitHub)" <gi...@apache.org> on 2025/04/08 15:23:10 UTC
[I] rdma握手失败：fail to read hello message (brpc)
liuxuting opened a new issue, #2944:
URL: https://github.com/apache/brpc/issues/2944
**Describe the bug (描述bug)**
背景：推理模型基于RDMA网络使用brpc进行节点间通信，业务节点单进程内启动多个brpc client和server，每个server监听在不同的端口。
问题：配置connect-type为single时，出现大量握手失败，rdma建链失败退化到tcp链接，通信时延明显增加。
配置connect-type为pooled时，启动阶段（大约持续1-2s）出现握手失败，报错包括rdma注册内存失败和fail to read hello message，后续握手成功正常进行rdma通信，时延随之降低。
**To Reproduce (复现方法)**
**Expected behavior (期望行为)**
**Versions (各种版本)**
OS: Linux version 5.10.134-13.an8.x86_64
Compiler: gcc version 8.5.0
brpc: 1.12
protobuf: baidu_std
**Additional context/screenshots (更多上下文/截图)**
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@brpc.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@brpc.apache.org
For additional commands, e-mail: dev-help@brpc.apache.org
Re: [I] rdma握手失败：fail to read hello message (brpc)
Posted by "yanglimingcn (via GitHub)" <gi...@apache.org>.
yanglimingcn commented on issue #2944:
URL: https://github.com/apache/brpc/issues/2944#issuecomment-2788332540
同一个进程的多个server，然后多个进程跑client，每个进程一个连接，做压测试，是什么效果呢？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@brpc.apache.org
For additional commands, e-mail: dev-help@brpc.apache.org
Re: [I] rdma通信失败：Memory not registered for RDMA (brpc)
Posted by "liuxuting (via GitHub)" <gi...@apache.org>.
liuxuting commented on issue #2944:
URL: https://github.com/apache/brpc/issues/2944#issuecomment-2862450698
client压测脚本：
#include <stdlib.h>
#include <unistd.h>
#include <vector>
#include <gflags/gflags.h>
#include "butil/atomicops.h"
#include "butil/fast_rand.h"
#include "butil/logging.h"
#include "brpc/rdma/rdma_helper.h"
#include "brpc/server.h"
#include "brpc/channel.h"
#include "bthread/bthread.h"
#include "bvar/latency_recorder.h"
#include "bvar/variable.h"
#include "test.pb.h"
#include <thread>
#include "butil/time.h"
#ifdef BRPC_WITH_RDMA
DEFINE_string(protocol, "baidu_std", "Protocol type.");
DEFINE_string(servers, "0.0.0.0:8002+0.0.0.0:8002", "IP Address of servers");
DEFINE_string(tcp_servers, "0.0.0.0:8002+0.0.0.0:8002", "IP Address of servers");
DEFINE_int32(rpc_timeout_ms, 2000, "RPC call timeout");
DEFINE_int32(port, 8002, "TCP Port of this server");
std::vector<std::string> g_servers;
std::vector<std::string> g_tcp_servers;
int rr_index = 0;
class PerformanceTest {
public:
PerformanceTest(int attachment_size, bool echo_attachment)
: _addr(NULL)
, _channel(NULL)
, _start_time(0)
, _iterations(0)
, _stop(false)
{
if (attachment_size > 0) {
_addr = malloc(attachment_size);
butil::fast_rand_bytes(_addr, attachment_size);
_attachment.append(_addr, attachment_size);
}
_echo_attachment = echo_attachment;
}
~PerformanceTest() {
if (_addr) {
free(_addr);
}
delete _channel;
}
inline bool IsStop() { return _stop; }
int Init(bool use_rdma) {
brpc::ChannelOptions options;
options.use_rdma = use_rdma;
options.protocol = FLAGS_protocol;
options.timeout_ms = FLAGS_rpc_timeout_ms;
options.max_retry = 1000;
std::string server;
if (use_rdma) {
server = g_servers[(rr_index++) % g_servers.size()];
} else {
server = g_tcp_servers[(rr_index++) % g_tcp_servers.size()];
}
_channel = new brpc::Channel();
if (_channel->Init(server.c_str(), &options) != 0) {
LOG(ERROR) << "Fail to initialize channel";
exit(1);
}
brpc::Controller cntl;
test::PerfTestResponse response;
test::PerfTestRequest request;
request.set_echo_attachment(_echo_attachment);
test::PerfTestService_Stub stub(_channel);
stub.Test(&cntl, &request, &response, NULL);
if (cntl.Failed()) {
LOG(ERROR) << "RPC call failed: " << cntl.ErrorText();
//return -1;
}
return 0;
}
struct RespClosure {
brpc::Controller* cntl;
test::PerfTestResponse* resp;
PerformanceTest* test;
};
void SendRequest() {
RespClosure* closure = new RespClosure;
test::PerfTestRequest request;
closure->resp = new test::PerfTestResponse();
closure->cntl = new brpc::Controller();
request.set_echo_attachment(_echo_attachment);
closure->cntl->request_attachment().append(_attachment);
closure->test = this;
google::protobuf::Closure* done = brpc::NewCallback(&HandleResponse, closure);
test::PerfTestService_Stub stub(_channel);
stub.Test(closure->cntl, &request, closure->resp, done);
}
static void HandleResponse(RespClosure* closure) {
std::unique_ptr<brpc::Controller> cntl_guard(closure->cntl);
std::unique_ptr<test::PerfTestResponse> response_guard(closure->resp);
if (closure->cntl->Failed()) {
LOG(ERROR) << "RPC call failed: " << closure->cntl->ErrorText();
closure->test->_stop = true;
return;
}
cntl_guard.reset(NULL);
response_guard.reset(NULL);
closure->test->SendRequest();
}
static void* RunTest(void* arg) {
PerformanceTest* test = (PerformanceTest*)arg;
test->SendRequest();
return NULL;
}
private:
void* _addr;
brpc::Channel* _channel;
uint64_t _start_time;
uint32_t _iterations;
volatile bool _stop;
butil::IOBuf _attachment;
bool _echo_attachment;
};
static void* DeleteTest(void* arg) {
PerformanceTest* test = (PerformanceTest*)arg;
delete test;
return NULL;
}
void Test(int thread_num, int attachment_size, bool use_rdma) {
std::vector<PerformanceTest*> tests;
for (int k = 0; k < thread_num; ++k) {
PerformanceTest* t = new PerformanceTest(attachment_size, true);
if (t->Init(use_rdma) < 0) {
exit(1);
}
tests.push_back(t);
}
uint64_t start_time = butil::gettimeofday_us();
bthread_t tid[thread_num];
for (int k = 0; k < thread_num; ++k) {
bthread_start_background(&tid[k], &BTHREAD_ATTR_NORMAL,
PerformanceTest::RunTest, tests[k]);
}
for (int k = 0; k < thread_num; ++k) {
while (!tests[k]->IsStop()) {
bthread_usleep(10000);
}
}
uint64_t end_time = butil::gettimeofday_us();
for (int k = 0; k < thread_num; ++k) {
bthread_start_background(&tid[k], &BTHREAD_ATTR_NORMAL, DeleteTest, tests[k]);
}
}
int client(int argc, char* argv[], bool use_rdma) {
GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
// Initialize RDMA environment in advance.
if (use_rdma) {
std::string::size_type pos1 = 0;
std::string::size_type pos2 = FLAGS_servers.find('+');
while (pos2 != std::string::npos) {
pos1 = pos2 + 1;
pos2 = FLAGS_servers.find('+', pos1);
}
g_servers.push_back(FLAGS_servers.substr(pos1));
} else {
std::string::size_type pos1 = 0;
std::string::size_type pos2 = FLAGS_tcp_servers.find('+');
while (pos2 != std::string::npos) {
pos1 = pos2 + 1;
pos2 = FLAGS_tcp_servers.find('+', pos1);
}
g_tcp_servers.push_back(FLAGS_tcp_servers.substr(pos1));
}
for (int j = 1; j <= 1024; j *= 4) {
for (int i = 1; i <= 16; i *= 2) {
Test(i, j, use_rdma);
}
}
return 0;
}
namespace test {
class PerfTestServiceImpl : public PerfTestService {
public:
PerfTestServiceImpl() {}
~PerfTestServiceImpl() {}
void Test(google::protobuf::RpcController* cntl_base,
const PerfTestRequest* request,
PerfTestResponse* response,
google::protobuf::Closure* done) {
brpc::ClosureGuard done_guard(done);
if (request->echo_attachment()) {
brpc::Controller* cntl =
static_cast<brpc::Controller*>(cntl_base);
cntl->response_attachment().append(cntl->request_attachment());
}
}
};
}
int main(int argc, char* argv[]) {
std::thread client_thread([&](){
client(argc, argv, true);
});
std::cout << "=====start client=====" << std::endl;
//std::this_thread::sleep_for(std::chrono::milliseconds(10000));
client_thread.detach();
std::thread client2_thread([&](){
client(argc, argv, false);
});
client2_thread.join();
}
#else
int main(int argc, char* argv[]) {
LOG(ERROR) << " brpc is not compiled with rdma. To enable it, please refer to https://github.com/apache/brpc/blob/master/docs/en/rdma.md";
return 0;
}
#endif
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For additional commands, e-mail: issues-help@brpc.apache.org
Re: [I] rdma通信失败：Memory not registered for RDMA (brpc)
Posted by "yanglimingcn (via GitHub)" <gi...@apache.org>.
yanglimingcn closed issue #2944: rdma通信失败：Memory not registered for RDMA
URL: https://github.com/apache/brpc/issues/2944
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For additional commands, e-mail: issues-help@brpc.apache.org
Re: [I] rdma通信失败：Memory not registered for RDMA (brpc)
Posted by "liuxuting (via GitHub)" <gi...@apache.org>.
liuxuting commented on issue #2944:
URL: https://github.com/apache/brpc/issues/2944#issuecomment-2865995897
> 我理解，rdma应该是全局生效的，部分server使用rdma部分server使用tcp感觉不太行，主要是那个rdma的内存是进程层面全局设置的。一部分不使用一部分使用感觉不太行。如果客户端是多进程，可以一些是use_rdma一些不使用rdma。
当前，对于单进程内混合tcp和rdma场景，我们略“粗暴”的tcp和rdma都是用BlockAllocate内存分配函数，并且将block都放在rdma注册的内存区域内，能够避免发生问题，时延也比较小。
blockmem_allocate作为全局的，会潜在的限制单个进程要么是use_rdma，要么是不使用rdma，无法混合。
咨询下社区后面会考虑更加细粒度的blockmem_allocate分配方式嘛？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For additional commands, e-mail: issues-help@brpc.apache.org
Re: [I] rdma通信失败：Memory not registered for RDMA (brpc)
Posted by "yanglimingcn (via GitHub)" <gi...@apache.org>.
yanglimingcn commented on issue #2944:
URL: https://github.com/apache/brpc/issues/2944#issuecomment-2872381551
感觉你现在这种使用方式没啥问题，allocate这块，不管是ptmalloc tcmalloc都是全局的，感觉按照线程区分，和当前的这些规则有点不符合，会改变架构，得仔细斟酌。
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For additional commands, e-mail: issues-help@brpc.apache.org
Re: [I] rdma通信失败：Memory not registered for RDMA (brpc)
Posted by "yanglimingcn (via GitHub)" <gi...@apache.org>.
yanglimingcn commented on issue #2944:
URL: https://github.com/apache/brpc/issues/2944#issuecomment-2864744613
我理解，rdma应该是全局生效的，部分server使用rdma部分server使用tcp感觉不太行，如果你希望某些server是tcp，可以关闭client端的rdma。
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For additional commands, e-mail: issues-help@brpc.apache.org
Re: [I] rdma通信失败：Memory not registered for RDMA (brpc)
Posted by "liuxuting (via GitHub)" <gi...@apache.org>.
liuxuting commented on issue #2944:
URL: https://github.com/apache/brpc/issues/2944#issuecomment-2862475350
server.cpp压测脚本：
#include <stdlib.h>
#include <unistd.h>
#include <vector>
#include <gflags/gflags.h>
#include "butil/atomicops.h"
#include "butil/fast_rand.h"
#include "butil/logging.h"
#include "brpc/rdma/rdma_helper.h"
#include "brpc/server.h"
#include "brpc/channel.h"
#include "bthread/bthread.h"
#include "bvar/latency_recorder.h"
#include "bvar/variable.h"
#include "test.pb.h"
#include <thread>
#include "butil/time.h"
#ifdef BRPC_WITH_RDMA
DEFINE_int32(attachment_size, -1, "Attachment size is used (in Bytes)");
DEFINE_bool(echo_attachment, false, "Select whether attachment should be echo");
DEFINE_string(connection_type, "single", "Connection type of the channel");
DEFINE_string(protocol, "baidu_std", "Protocol type.");
DEFINE_string(server_addr, "0.0.0.0:8002", "Address of server");
DEFINE_string(server_addr_tcp, "0.0.0.0:8002", "Address of server");
butil::atomic<uint64_t> g_last_time(0);
namespace test {
class PerfTestServiceImpl : public PerfTestService {
public:
PerfTestServiceImpl() {}
~PerfTestServiceImpl() {}
void Test(google::protobuf::RpcController* cntl_base,
const PerfTestRequest* request,
PerfTestResponse* response,
google::protobuf::Closure* done) {
brpc::ClosureGuard done_guard(done);
uint64_t last = g_last_time.load(butil::memory_order_relaxed);
uint64_t now = butil::monotonic_time_us();
if (now > last && now - last > 100000) {
if (g_last_time.exchange(now, butil::memory_order_relaxed) == last) {
response->set_cpu_usage(bvar::Variable::describe_exposed("process_cpu_usage"));
} else {
response->set_cpu_usage("");
}
} else {
response->set_cpu_usage("");
}
if (request->echo_attachment()) {
brpc::Controller* cntl =
static_cast<brpc::Controller*>(cntl_base);
cntl->response_attachment().append(cntl->request_attachment());
}
}
};
}
int server(int argc, char* argv[], bool use_rdma) {
GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
brpc::Server server;
test::PerfTestServiceImpl perf_test_service_impl;
if (server.AddService(&perf_test_service_impl,
brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
LOG(ERROR) << "Fail to add service";
return -1;
}
g_last_time.store(0, butil::memory_order_relaxed);
brpc::ServerOptions options;
options.use_rdma = use_rdma;
if (use_rdma) {
if (server.Start(FLAGS_server_addr.c_str(), &options) != 0) {
LOG(ERROR) << "Fail to start EchoServer";
return -1;
}
} else {
if (server.Start(FLAGS_server_addr_tcp.c_str(), &options) != 0) {
LOG(ERROR) << "Fail to start EchoServer";
return -1;
}
}
server.RunUntilAskedToQuit();
return 0;
}
int main(int argc, char* argv[]) {
std::thread server_thread([&]()
{
server(argc, argv, false);
});
std::cout << "=====start server2=====" << std::endl;
std::this_thread::sleep_for(std::chrono::milliseconds(10000));
server_thread.detach();
std::thread server2_thread([&](){
server(argc, argv, true);
});
server2_thread.join();
}
#else
int main(int argc, char* argv[]) {
LOG(ERROR) << " brpc is not compiled with rdma. To enable it, please refer to https://github.com/apache/brpc/blob/master/docs/en/rdma.md";
return 0;
}
#endif
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For additional commands, e-mail: issues-help@brpc.apache.org
Re: [I] rdma通信失败：Memory not registered for RDMA (brpc)
Posted by "kangshantong (via GitHub)" <gi...@apache.org>.
kangshantong commented on issue #2944:
URL: https://github.com/apache/brpc/issues/2944#issuecomment-2953814778
> 我理解，rdma应该是全局生效的，部分server使用rdma部分server使用tcp感觉不太行，主要是那个rdma的内存是进程层面全局设置的。一部分不使用一部分使用感觉不太行。如果客户端是多进程，可以一些是use_rdma一些不使用rdma。
启动server和client的时候，有个use_rdma选项。在一个有两张网卡(一张tcp网卡，一张rdma网卡)机器上，同一个进程启动两个brpc Server，一个打开use_rdma, 一个关闭use_rdma，这种情形，目前brpc是支持的吗
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For additional commands, e-mail: issues-help@brpc.apache.org
Re: [I] rdma通信失败：Memory not registered for RDMA (brpc)
Posted by "liuxuting (via GitHub)" <gi...@apache.org>.
liuxuting commented on issue #2944:
URL: https://github.com/apache/brpc/issues/2944#issuecomment-2862480017
test.proto：
`syntax="proto2";
option cc_generic_services = true;
package test;
message PerfTestRequest {
required bool echo_attachment = 1;
};
message PerfTestResponse {
required string cpu_usage = 1;
};
service PerfTestService {
rpc Test(PerfTestRequest) returns (PerfTestResponse);
};
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@brpc.apache.org
For additional commands, e-mail: issues-help@brpc.apache.org