(trafficserver) branch master updated: Handle OpenSSL 4 handshake buffer exhaustion (#13761)

[bn...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

bneradt pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git


The following commit(s) were added to refs/heads/master by this push:
     new fe7cfc9427 Handle OpenSSL 4 handshake buffer exhaustion (#13761)
fe7cfc9427 is described below

commit fe7cfc94278d05e0eb772f08a14628a0141437e3
Author: Brian Neradt <br...@gmail.com>
AuthorDate: Thu Oct 1 13:10:35 2026 -0500

    Handle OpenSSL 4 handshake buffer exhaustion (#13761)
    
    TLS handshakes stall with OpenSSL 4 because an empty memory BIO no
    longer reports EOF when its empty-read behavior requests a retry. ATS
    therefore retains the exhausted handshake buffer and never resumes
    reading the client connection.
    
    This patch checks the pending byte count to detect exhausted handshake
    buffers and resume socket reads. This check also works with earlier
    OpenSSL releases and removes the unused BoringSSL EOF fallback.
    
    Co-authored-by: Codex GPT-6.1 Sol High
---
 src/iocore/net/SSLNetVConnection.cc | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/iocore/net/SSLNetVConnection.cc b/src/iocore/net/SSLNetVConnection.cc
index 01d3573c94..bda5250258 100644
--- a/src/iocore/net/SSLNetVConnection.cc
+++ b/src/iocore/net/SSLNetVConnection.cc
@@ -56,11 +56,6 @@
 
 using namespace std::literals;
 
-// This is missing from BoringSSL
-#ifndef BIO_eof
-#define BIO_eof(b) (int)BIO_ctrl(b, BIO_CTRL_EOF, 0, nullptr)
-#endif
-
 #define SSL_READ_ERROR_NONE        0
 #define SSL_READ_ERROR             1
 #define SSL_READ_READY             2
@@ -457,7 +452,8 @@ bool
 SSLNetVConnection::update_rbio(bool move_to_socket)
 {
   bool retval = false;
-  if (BIO_eof(SSL_get_rbio(this->ssl)) && this->handShakeReader != nullptr) {
+  // OpenSSL 4 does not report EOF for an empty memory BIO configured to retry reads.
+  if (this->handShakeReader != nullptr && BIO_ctrl_pending(SSL_get_rbio(this->ssl)) == 0) {
     Dbg(dbg_ctl_ssl, "Consuming handShakeBioStored=%d bytes from the handshake reader", this->handShakeBioStored);
     this->handShakeReader->consume(this->handShakeBioStored);
     this->handShakeBioStored = 0;
@@ -1354,8 +1350,8 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err)
   // We only feed CLIENT_HELLO bytes into our temporary buffers. If we are past
   // the CLIENT_HELLO, then no need to buffer.
   if (in_client_hello && this->handShakeReader) {
-    if (BIO_eof(SSL_get_rbio(this->ssl))) { // No more data in the buffer
-                                            // Is this the first read?
+    if (BIO_ctrl_pending(SSL_get_rbio(this->ssl)) == 0) { // No more data in the buffer
+                                                          // Is this the first read?
 #if TS_USE_TLS_ASYNC
       if (SSLConfigParams::async_handshake_enabled) {
         SSL_set_mode(ssl, SSL_MODE_ASYNC);