You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2026/10/21 06:19:24 UTC

(arrow) branch main updated: GH-49818: [C++][FlightRPC] Don't call Finish() twice on handshake stream (#54685)

This is an automated email from the ASF dual-hosted git repository.

kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/main by this push:
     new 14daca685ee GH-49818: [C++][FlightRPC] Don't call Finish() twice on handshake stream (#54685)
14daca685ee is described below

commit 14daca685eecde57ae688a427fcdec210f8544a8
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Wed Oct 21 15:19:16 2026 +0900

    GH-49818: [C++][FlightRPC] Don't call Finish() twice on handshake stream (#54685)
    
    ### Rationale for this change
    
    `arrow-flight-sql-odbc-test` sometimes aborts with `GRPC_CALL_ERROR_TOO_MANY_OPERATIONS` in `ClientReaderWriter::Finish()` of the handshake stream.
    
    This happens when a server finishes a handshake before the client writes or reads. For example, `NoOpAuthHandler` on the server side and Dremio do this. In this case:
    
    1. `GrpcClientAuthSender::Write()` (or `GrpcClientAuthReader::Read()`) fails and calls `Finish()` on the handshake stream.
    2. The returned status is OK, so the client auth handler and `AuthenticateInternal()` continue.
    3. `AuthenticateInternal()` calls `WritesDone()`, drains the stream and calls `Finish()` again.
    4. gRPC rejects the second `RECV_STATUS_ON_CLIENT` with `GRPC_CALL_ERROR_TOO_MANY_OPERATIONS` and aborts the process.
    
    `GRPC_TRACE=api` shows this sequence:
    
    ```text
    SEND_MESSAGE                                   -> ERROR: Attempt to send message after stream was closed.
    RECV_INITIAL_METADATA + RECV_STATUS_ON_CLIENT  -> OK  (Finish() by GrpcClientAuthSender::Write())
    SEND_CLOSE_FROM_CLIENT                                (WritesDone())
    RECV_MESSAGE                                          (drain)
    RECV_STATUS_ON_CLIENT                          -> GRPC_CALL_ERROR_TOO_MANY_OPERATIONS (Finish() by AuthenticateInternal())
    ```
    
    Whether the server finishes before the client writes is a race. So it's flaky.
    
    ### What changes are included in this PR?
    
    * Add `GrpcClientHandshakeStream` that wraps the handshake stream:
      * `Finish()` is called at most once. The second and later calls return the status of the first call.
      * `Read()`, `Write()` and `WritesDone()` return `false` without calling gRPC after the stream is finished.
    * Return the status in `AuthenticateInternal()` without `WritesDone()` and draining when the stream is already finished.
    
    ### Are these changes tested?
    
    Yes.
    
    ### Are there any user-facing changes?
    
    Yes. Flight clients don't abort the process when a server finishes a handshake early.
    
    ### Was AI used for this PR?
    
    In accordance to the [AI generation guidelines](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code), please disclose below whether and how AI was used in this PR.
    
    **PR code and description written by:**
    
    - [x] Human
    - [x] AI
    
    **Reviewed before submission by:**
    
    - [x] Human
    - [x] AI
    - [ ] Not reviewed
    
    * GitHub Issue: #49818
    
    Authored-by: Sutou Kouhei <ko...@clear-code.com>
    Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
 cpp/src/arrow/flight/flight_test.cc                | 47 +++++++++++++++
 cpp/src/arrow/flight/transport/grpc/grpc_client.cc | 70 +++++++++++++++-------
 2 files changed, 97 insertions(+), 20 deletions(-)

diff --git a/cpp/src/arrow/flight/flight_test.cc b/cpp/src/arrow/flight/flight_test.cc
index eb931011906..ab74e1fd6b8 100644
--- a/cpp/src/arrow/flight/flight_test.cc
+++ b/cpp/src/arrow/flight/flight_test.cc
@@ -453,6 +453,47 @@ class TestBasicAuthHandler : public ::testing::Test {
   std::unique_ptr<FlightServerBase> server_;
 };
 
+// A client auth handler that reads and writes after the server
+// finishes the handshake.
+class ReadWriteAfterFinishClientAuthHandler : public ClientAuthHandler {
+ public:
+  Status Authenticate(ClientAuthSender* outgoing, ClientAuthReader* incoming) override {
+    std::string token;
+    // The server finishes the handshake without sending anything. So
+    // this finishes the stream.
+    RETURN_NOT_OK(incoming->Read(&token));
+    // The stream is already finished.
+    return outgoing->Write("token");
+  }
+
+  Status GetToken(std::string* token) override {
+    *token = "";
+    return Status::OK();
+  }
+};
+
+class TestNoOpAuthHandler : public ::testing::Test {
+ public:
+  void SetUp() {
+    ASSERT_OK(MakeServer<AuthTestServer>(
+        &server_, &client_,
+        [](FlightServerOptions* options) {
+          options->auth_handler = std::make_unique<NoOpAuthHandler>();
+          return Status::OK();
+        },
+        [](FlightClientOptions* options) { return Status::OK(); }));
+  }
+
+  void TearDown() {
+    ASSERT_OK(client_->Close());
+    ASSERT_OK(server_->Shutdown());
+  }
+
+ protected:
+  std::unique_ptr<FlightClient> client_;
+  std::unique_ptr<FlightServerBase> server_;
+};
+
 class TestTls : public ::testing::Test {
  public:
   void SetUp() {
@@ -1325,6 +1366,12 @@ TEST_F(TestBasicAuthHandler, CheckPeerIdentity) {
   ASSERT_EQ(result->body->ToString(), "user");
 }
 
+TEST_F(TestNoOpAuthHandler, ReadWriteAfterFinish) {
+  // GH-49818: This must not call Finish() on the handshake stream twice.
+  ASSERT_OK(client_->Authenticate(
+      {}, std::make_unique<ReadWriteAfterFinishClientAuthHandler>()));
+}
+
 TEST_F(TestTls, DoAction) {
   FlightCallOptions options;
   options.timeout = TimeoutDuration{5.0};
diff --git a/cpp/src/arrow/flight/transport/grpc/grpc_client.cc b/cpp/src/arrow/flight/transport/grpc/grpc_client.cc
index f1a5dcd6863..5f2a2aa0a0d 100644
--- a/cpp/src/arrow/flight/transport/grpc/grpc_client.cc
+++ b/cpp/src/arrow/flight/transport/grpc/grpc_client.cc
@@ -214,13 +214,47 @@ class GrpcClientInterceptorAdapterFactory
   std::vector<std::shared_ptr<ClientMiddlewareFactory>> middleware_;
 };
 
-class GrpcClientAuthSender : public ClientAuthSender {
+/// \brief A Handshake stream that calls Finish() at most once.
+///
+/// gRPC aborts the process if Finish() is called twice on the same call.
+class GrpcClientHandshakeStream {
  public:
-  explicit GrpcClientAuthSender(
+  explicit GrpcClientHandshakeStream(
       std::shared_ptr<
           ::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
           stream)
-      : stream_(stream) {}
+      : stream_(std::move(stream)) {}
+
+  bool finished() const { return finished_; }
+
+  bool Write(const pb::HandshakeRequest& request) {
+    return !finished_ && stream_->Write(request);
+  }
+
+  bool Read(pb::HandshakeResponse* response) {
+    return !finished_ && stream_->Read(response);
+  }
+
+  bool WritesDone() { return !finished_ && stream_->WritesDone(); }
+
+  const ::grpc::Status& Finish() {
+    if (!finished_) {
+      status_ = stream_->Finish();
+      finished_ = true;
+    }
+    return status_;
+  }
+
+ private:
+  std::shared_ptr<::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
+      stream_;
+  bool finished_ = false;
+  ::grpc::Status status_;
+};
+
+class GrpcClientAuthSender : public ClientAuthSender {
+ public:
+  explicit GrpcClientAuthSender(GrpcClientHandshakeStream* stream) : stream_(stream) {}
 
   Status Write(const std::string& token) override {
     pb::HandshakeRequest response;
@@ -232,17 +266,12 @@ class GrpcClientAuthSender : public ClientAuthSender {
   }
 
  private:
-  std::shared_ptr<::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
-      stream_;
+  GrpcClientHandshakeStream* stream_;
 };
 
 class GrpcClientAuthReader : public ClientAuthReader {
  public:
-  explicit GrpcClientAuthReader(
-      std::shared_ptr<
-          ::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
-          stream)
-      : stream_(stream) {}
+  explicit GrpcClientAuthReader(GrpcClientHandshakeStream* stream) : stream_(stream) {}
 
   Status Read(std::string* token) override {
     pb::HandshakeResponse request;
@@ -254,8 +283,7 @@ class GrpcClientAuthReader : public ClientAuthReader {
   }
 
  private:
-  std::shared_ptr<::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
-      stream_;
+  GrpcClientHandshakeStream* stream_;
 };
 
 /// \brief The base of the ClientDataStream implementation for gRPC.
@@ -1065,16 +1093,18 @@ class GrpcClientImpl : public internal::ClientTransport {
 
  private:
   Status AuthenticateInternal(ClientRpc& rpc) {
-    std::shared_ptr<
-        ::grpc::ClientReaderWriter<pb::HandshakeRequest, pb::HandshakeResponse>>
-        stream = stub_->Handshake(&rpc.context);
+    GrpcClientHandshakeStream stream{stub_->Handshake(&rpc.context)};
     if (auth_handler_) {
-      GrpcClientAuthSender outgoing{stream};
-      GrpcClientAuthReader incoming{stream};
+      GrpcClientAuthSender outgoing{&stream};
+      GrpcClientAuthReader incoming{&stream};
       RETURN_NOT_OK(auth_handler_->Authenticate(&outgoing, &incoming));
     }
+    // The server may have finished the handshake while authenticating
+    if (stream.finished()) {
+      return FromGrpcStatus(stream.Finish(), &rpc.context);
+    }
     // Explicitly close our side of the connection
-    bool finished_writes = stream->WritesDone();
+    bool finished_writes = stream.WritesDone();
     if (!finished_writes) {
       return MakeFlightError(FlightStatusCode::Internal,
                              "Could not finish writing before closing");
@@ -1084,9 +1114,9 @@ class GrpcClientImpl : public internal::ClientTransport {
     // reader finishes, so it's OK to assume the client no longer
     // wants to read and drain the read side.
     pb::HandshakeResponse response;
-    while (stream->Read(&response)) {
+    while (stream.Read(&response)) {
     }
-    RETURN_NOT_OK(FromGrpcStatus(stream->Finish(), &rpc.context));
+    RETURN_NOT_OK(FromGrpcStatus(stream.Finish(), &rpc.context));
     return Status::OK();
   }