From 4afac8ee3a6adda1165423bccdfe71fbf97d6ae2 Mon Sep 17 00:00:00 2001 From: Travis Downs Date: Tue, 17 Nov 2026 17:22:18 -0300 Subject: [PATCH] net/tls: keep the first handshake error for queued handshake waiters A read and a write on a TLS socket that find the session not yet connected both call handshake(), which serializes them on the input and output semaphores. If the first handshake (driven by the winner of the race) failed, the second ran it again on the failed session. The library then reported a different error, often a generic one ("System error during handshake" in OpenSSL, "The specified session has been invalidated" in GnuTLS), and it replaced the recorded error, so the second waiter and every later operation saw it instead of the real cause, such as a certificate verification failure. Instead, check the error after acquiring the semaphore, and if set return the recorded error instead of retrying the handshake, in both backends. (cherry picked from commit 46f0a47a25a85fd41b7a6d844370cd3157e01697) --- src/net/tls_gnutls.cc | 7 +++ src/net/tls_openssl.cc | 7 +++ tests/unit/tls_test.cc | 108 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+) diff --git a/src/net/tls_gnutls.cc b/src/net/tls_gnutls.cc index 4daa85c7464..a344717eab9 100644 --- a/src/net/tls_gnutls.cc +++ b/src/net/tls_gnutls.cc @@ -723,6 +723,13 @@ class session : public enable_shared_from_this, public tls::session_imp // acquire both semaphores to sync both read & write return with_semaphore(_in_sem, 1, [this, func] { return with_semaphore(_out_sem, 1, [this, func] { + // check error again after waiting on the semaphore + // to avoid a 2nd handshake attempt when there are multiple + // callers of handshake (the libraries reject this with a + // different error) + if (_error) { + return make_exception_future<>(_error); + } return std::invoke(func, this).handle_exception([this](auto ep) { if (!_error) { _error = ep; diff --git a/src/net/tls_openssl.cc b/src/net/tls_openssl.cc index 4ac9a9ed66d..55f14e590b1 100644 --- a/src/net/tls_openssl.cc +++ b/src/net/tls_openssl.cc @@ -1544,6 +1544,13 @@ class openssl_session : public enable_shared_from_this, public return with_semaphore(_in_sem, 1, [this] { return with_semaphore(_out_sem, 1, [this] { + // check error again after waiting on the semaphore + // to avoid a 2nd handshake attempt when there are multiple + // callers of handshake (the libraries reject this with a + // different error) + if (_error) { + return make_exception_future<>(_error); + } return do_handshake().handle_exception([this](auto ep) { if (!_error) { _error = ep; diff --git a/tests/unit/tls_test.cc b/tests/unit/tls_test.cc index 834ebde818c..bd864c237f2 100644 --- a/tests/unit/tls_test.cc +++ b/tests/unit/tls_test.cc @@ -2767,3 +2767,111 @@ SEASTAR_THREAD_TEST_CASE(test_handshake_error_without_alert_is_not_from_peer) { require_contains(err, "Received HTTP request on HTTPS server"); require_not_contains(err, "Received TLS alert from peer"); } + + +// Starts a flush and then a read on a client whose handshake will fail, so the +// read queues behind the handshake that the flush starts. Returns the error +// each one fails with. +static std::pair concurrent_flush_and_read_errors(streams& client) { + auto write = client.out.write(message).then([&client] { return client.out.flush(); }); + auto read = client.in.read().discard_result(); + + std::exception_ptr write_error; + std::exception_ptr read_error; + try { + write.get(); + } catch (...) { + write_error = std::current_exception(); + } + try { + read.get(); + } catch (...) { + read_error = std::current_exception(); + } + BOOST_REQUIRE_MESSAGE(write_error, "Expected the flush to fail"); + BOOST_REQUIRE_MESSAGE(read_error, "Expected the read to fail"); + return {write_error, read_error}; +} + +static sstring what(std::exception_ptr ep) { + try { + std::rethrow_exception(ep); + } catch (const std::exception& e) { + return e.what(); + } +} + +// A read that waits on the handshake a flush started must fail with that +// handshake's error, not with the result of attempting the handshake again. +SEASTAR_THREAD_TEST_CASE(test_read_queued_behind_handshake_gets_verification_error) { + tls::credentials_builder cb; + // A trust store that does not contain the server's CA. + cb.set_x509_trust_file(certfile("tls-ca-bundle.pem"), tls::x509_crt_format::PEM).get(); + auto client_creds = cb.build_certificate_credentials(); + + tls::credentials_builder sb; + sb.set_x509_key_file(certfile("test.crt"), certfile("test.key"), tls::x509_crt_format::PEM).get(); + sb.set_dh_level(); + auto server_creds = sb.build_server_credentials(); + + auto b1 = ::make_lw_shared(nullptr, loopback_buffer::type::SERVER_TX); + auto b2 = ::make_lw_shared(nullptr, loopback_buffer::type::CLIENT_TX); + auto ssi = std::make_unique(b1, b2); + auto csi = std::make_unique(b2, b1); + + auto ss = tls::wrap_server(server_creds, connected_socket(std::move(ssi))).get(); + auto cs = tls::wrap_client(client_creds, connected_socket(std::move(csi)), + tls::tls_options{.server_name = "test.scylladb.org"}).get(); + + auto server_in = ss.input(); + auto server_loop = server_in.read().discard_result().handle_exception([](std::exception_ptr) {}); + + streams client(std::move(cs)); + auto [write_error, read_error] = concurrent_flush_and_read_errors(client); + BOOST_REQUIRE_THROW(std::rethrow_exception(write_error), tls::verification_error); + BOOST_REQUIRE_THROW(std::rethrow_exception(read_error), tls::verification_error); + + b1->abort(); + b2->abort(); + server_loop.get(); + client.out.close().handle_exception([](std::exception_ptr) {}).get(); + client.in.close().get(); + server_in.close().get(); +} + +// Same as above, for a handshake that the TLS library itself fails, here +// because the peer answers with plain HTTP. +SEASTAR_THREAD_TEST_CASE(test_read_queued_behind_handshake_gets_handshake_error) { + tls::credentials_builder cb; + cb.set_x509_trust_file(certfile("catest.pem"), tls::x509_crt_format::PEM).get(); + auto client_creds = cb.build_certificate_credentials(); + + auto b1 = ::make_lw_shared(nullptr, loopback_buffer::type::SERVER_TX); + auto b2 = ::make_lw_shared(nullptr, loopback_buffer::type::CLIENT_TX); + auto ssi = std::make_unique(b1, b2); + auto csi = std::make_unique(b2, b1); + + ::connected_socket peer(std::move(ssi)); + auto peer_in = peer.input(); + output_stream peer_out(peer.output().detach(), 1024); + auto peer_loop = peer_in.read().then([&peer_out](temporary_buffer) { + return peer_out.write(sstring("HTTP/1.1 400 Bad Request\r\n\r\n")).then([&peer_out] { + return peer_out.flush(); + }); + }).handle_exception([](std::exception_ptr) {}); + + auto cs = tls::wrap_client(client_creds, connected_socket(std::move(csi)), + tls::tls_options{.server_name = "test.scylladb.org"}).get(); + streams client(std::move(cs)); + auto [write_error, read_error] = concurrent_flush_and_read_errors(client); + BOOST_TEST_MESSAGE(fmt::format("flush error: {}", what(write_error)).c_str()); + BOOST_REQUIRE_EQUAL(what(read_error), what(write_error)); + + b1->abort(); + b2->abort(); + peer_loop.get(); + client.out.close().handle_exception([](std::exception_ptr) {}).get(); + client.in.close().get(); + peer_out.close().handle_exception([](std::exception_ptr) {}).get(); + peer_in.close().get(); +}