You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2026/10/21 06:19:24 UTC
(arrow) branch main updated: GH-49818: [C++][FlightRPC] Don't call Finish() twice on handshake stream (#54685)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 14daca685ee GH-49818: [C++][FlightRPC] Don't call Finish() twice on handshake stream (#54685)
14daca685ee is described below
commit 14daca685eecde57ae688a427fcdec210f8544a8
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Wed Oct 21 15:19:16 2026 +0900
GH-49818: [C++][FlightRPC] Don't call Finish() twice on handshake stream (#54685)
### Rationale for this change
`arrow-flight-sql-odbc-test` sometimes aborts with `GRPC_CALL_ERROR_TOO_MANY_OPERATIONS` in `ClientReaderWriter::Finish()` of the handshake stream.
This happens when a server finishes a handshake before the client writes or reads. For example, `NoOpAuthHandler` on the server side and Dremio do this. In this case:
1. `GrpcClientAuthSender::Write()` (or `GrpcClientAuthReader::Read()`) fails and calls `Finish()` on the handshake stream.
2. The returned status is OK, so the client auth handler and `AuthenticateInternal()` continue.
3. `AuthenticateInternal()` calls `WritesDone()`, drains the stream and calls `Finish()` again.
4. gRPC rejects the second `RECV_STATUS_ON_CLIENT` with `GRPC_CALL_ERROR_TOO_MANY_OPERATIONS` and aborts the process.
`GRPC_TRACE=api` shows this sequence:
```text
SEND_MESSAGE -> ERROR: Attempt to send message after stream was closed.
RECV_INITIAL_METADATA + RECV_STATUS_ON_CLIENT -> OK (Finish() by GrpcClientAuthSender::Write())
SEND_CLOSE_FROM_CLIENT (WritesDone())
RECV_MESSAGE (drain)
RECV_STATUS_ON_CLIENT -> GRPC_CALL_ERROR_TOO_MANY_OPERATIONS (Finish() by AuthenticateInternal())
```
Whether the server finishes before the client writes is a race. So it's flaky.
### What changes are included in this PR?
* Add `GrpcClientHandshakeStream` that wraps the handshake stream:
* `Finish()` is called at most once. The second and later calls return the status of the first call.
* `Read()`, `Write()` and `WritesDone()` return `false` without calling gRPC after the stream is finished.
* Return the status in `AuthenticateInternal()` without `WritesDone()` and draining when the stream is already finished.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
Yes. Flight clients don't abort the process when a server finishes a handshake early.
### Was AI used for this PR?
In accordance to the [AI generation guidelines](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code), please disclose below whether and how AI was used in this PR.
**PR code and description written by:**
- [x] Human
- [x] AI
**Reviewed before submission by:**
- [x] Human
- [x] AI
- [ ] Not reviewed
* GitHub Issue: #49818
Authored-by: Sutou Kouhei <ko...@clear-code.com>
Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
cpp/src/arrow/flight/flight_test.cc | 47 +++++++++++++++
cpp/src/arrow/flight/transport/grpc/grpc_client.cc | 70 +++++++++++++++-------
2 files changed, 97 insertions(+), 20 deletions(-)
diff --git a/cpp/src/arrow/flight/flight_test.cc b/cpp/src/arrow/flight/flight_test.cc
index eb931011906..ab74e1fd6b8 100644
--- a/cpp/src/arrow/flight/flight_test.cc
+++ b/cpp/src/arrow/flight/flight_test.cc
@@ -453,6 +453,47 @@ class TestBasicAuthHandler : public ::testing::Test {
std::unique_ptr<FlightServerBase> server_;
};
+// A client auth handler that reads and writes after the server
+// finishes the handshake.
+class ReadWriteAfterFinishClientAuthHandler : public ClientAuthHandler {
+ public:
+ Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override {
+ std::string token;
+ // The server finishes the handshake without sending anything. So
+ // this finishes the stream.
+ RETURN_NOT_OK(incoming->Read(&token));
+ // The stream is already finished.
+ return outgoing->Write("token");
+ }
+
+ Status GetToken(std::string* token) override {
+ *token = "";
+ return Status::OK();
+ }
+};
+
+class TestNoOpAuthHandler : public ::testing::Test {
+ public:
+ void SetUp() {
+ ASSERT_OK(MakeServer<AuthTestServer>(
+ &server_, &client_,
+ [](FlightServerOptions* options) {
+ options->auth_handler = std::make_unique<NoOpAuthHandler>();
+ return Status::OK();
+ },
+ [](FlightClientOptions* options) { return Status::OK(); }));
+ }
+
+ void TearDown() {
+ ASSERT_OK(client_->Close());
+ ASSERT_OK(server_->Shutdown());
+ }
+
+ protected:
+ std::unique_ptr<FlightClient> client_;
+ std::unique_ptr<FlightServerBase> server_;
+};
+
class TestTls : public ::testing::Test {
public:
void SetUp() {
@@ -1325,6 +1366,12 @@ TEST_F(TestBasicAuthHandler, CheckPeerIdentity) {
ASSERT_EQ(result->body->ToString(), "user");
}
+TEST_F(TestNoOpAuthHandler, ReadWriteAfterFinish) {
+ // GH-49818: This must not call Finish() on the handshake stream twice.
+ ASSERT_OK(client_->Authenticate(
+ {}, std::make_unique<ReadWriteAfterFinishClientAuthHandler>()));
+}
+
TEST_F(TestTls, DoAction) {
FlightCallOptions options;
options.timeout = TimeoutDuration{5.0};
diff --git a/cpp/src/arrow/flight/transport/grpc/grpc_client.cc b/cpp/src/arrow/flight/transport/grpc/grpc_client.cc
index f1a5dcd6863..5f2a2aa0a0d 100644
--- a/cpp/src/arrow/flight/transport/grpc/grpc_client.cc
+++ b/cpp/src/arrow/flight/transport/grpc/grpc_client.cc
@@ -214,13 +214,47 @@ class GrpcClientInterceptorAdapterFactory
std::vector<std::shared_ptr<ClientMiddlewareFactory>> middleware_;
};
-class GrpcClientAuthSender : public ClientAuthSender {
+/// \brief A Handshake stream that calls Finish() at most once.
+///
+/// gRPC aborts the process if Finish() is called twice on the same call.
+class GrpcClientHandshakeStream {
public:
- explicit GrpcClientAuthSender(
+ explicit GrpcClientHandshakeStream(
std::shared_ptr<
::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
stream)
- : stream_(stream) {}
+ : stream_(std::move(stream)) {}
+
+ bool finished() const { return finished_; }
+
+ bool Write(const pb::HandshakeRequest& request) {
+ return !finished_ && stream_->Write(request);
+ }
+
+ bool Read(pb::HandshakeResponse* response) {
+ return !finished_ && stream_->Read(response);
+ }
+
+ bool WritesDone() { return !finished_ && stream_->WritesDone(); }
+
+ const ::grpc::Status& Finish() {
+ if (!finished_) {
+ status_ = stream_->Finish();
+ finished_ = true;
+ }
+ return status_;
+ }
+
+ private:
+ std::shared_ptr<::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
+ stream_;
+ bool finished_ = false;
+ ::grpc::Status status_;
+};
+
+class GrpcClientAuthSender : public ClientAuthSender {
+ public:
+ explicit GrpcClientAuthSender(GrpcClientHandshakeStream* stream) : stream_(stream) {}
Status Write(const std::string& token) override {
pb::HandshakeRequest response;
@@ -232,17 +266,12 @@ class GrpcClientAuthSender : public ClientAuthSender {
}
private:
- std::shared_ptr<::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
- stream_;
+ GrpcClientHandshakeStream* stream_;
};
class GrpcClientAuthReader : public ClientAuthReader {
public:
- explicit GrpcClientAuthReader(
- std::shared_ptr<
- ::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
- stream)
- : stream_(stream) {}
+ explicit GrpcClientAuthReader(GrpcClientHandshakeStream* stream) : stream_(stream) {}
Status Read(std::string* token) override {
pb::HandshakeResponse request;
@@ -254,8 +283,7 @@ class GrpcClientAuthReader : public ClientAuthReader {
}
private:
- std::shared_ptr<::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
- stream_;
+ GrpcClientHandshakeStream* stream_;
};
/// \brief The base of the ClientDataStream implementation for gRPC.
@@ -1065,16 +1093,18 @@ class GrpcClientImpl : public internal::ClientTransport {
private:
Status AuthenticateInternal(ClientRpc& rpc) {
- std::shared_ptr<
- ::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
- stream = stub_->Handshake(&rpc.context);
+ GrpcClientHandshakeStream stream{stub_->Handshake(&rpc.context)};
if (auth_handler_) {
- GrpcClientAuthSender outgoing{stream};
- GrpcClientAuthReader incoming{stream};
+ GrpcClientAuthSender outgoing{&stream};
+ GrpcClientAuthReader incoming{&stream};
RETURN_NOT_OK(auth_handler_->Authenticate(&outgoing, &incoming));
}
+ // The server may have finished the handshake while authenticating
+ if (stream.finished()) {
+ return FromGrpcStatus(stream.Finish(), &rpc.context);
+ }
// Explicitly close our side of the connection
- bool finished_writes = stream->WritesDone();
+ bool finished_writes = stream.WritesDone();
if (!finished_writes) {
return MakeFlightError(FlightStatusCode::Internal,
"Could not finish writing before closing");
@@ -1084,9 +1114,9 @@ class GrpcClientImpl : public internal::ClientTransport {
// reader finishes, so it's OK to assume the client no longer
// wants to read and drain the read side.
pb::HandshakeResponse response;
- while (stream->Read(&response)) {
+ while (stream.Read(&response)) {
}
- RETURN_NOT_OK(FromGrpcStatus(stream->Finish(), &rpc.context));
+ RETURN_NOT_OK(FromGrpcStatus(stream.Finish(), &rpc.context));
return Status::OK();
}