PoC: low level TLS machine

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs

@@ -688,6 +688,15 @@ internal static SecurityStatusPalErrorCode DoSslHandshake(SafeSslHandle context,
                     return SecurityStatusPalErrorCode.CredentialsNeeded;
                 }
 
+                if (errorCode == Ssl.SslErrorCode.SSL_ERROR_WANT_RETRY_VERIFY)
+                {
+                    // OpenSSL 3.0+ retry-verify: the certificate verification
+                    // callback paused the handshake. The application owns
+                    // certificate validation and must resume the handshake
+                    // (by calling DoSslHandshake again) once it has a verdict.
+                    return SecurityStatusPalErrorCode.NeedsRemoteCertificateValidation;
+                }
+
                 if (errorCode == Ssl.SslErrorCode.SSL_ERROR_SSL && context.CertificateValidationException is Exception ex)
                 {
                     // Clear the OpenSSL error queue since we are using our own
@@ -874,7 +883,25 @@ internal static int CertVerifyCallback(IntPtr storeCtx, IntPtr arg)
                     .TryGetTarget(out SslAuthenticationOptions? options);
                 Debug.Assert(options != null, "Expected to get SslAuthenticationOptions from GCHandle");
 
-                sslHandle = (SafeSslHandle)options!.SslStream!._securityContext!;
+                sslHandle = options!.SafeSslHandle as SafeSslHandle;
+                Debug.Assert(sslHandle is not null, "Expected SslAuthenticationOptions.SafeSslHandle to be set by SafeSslHandle.Create");
+
+                // External-validation fast path: when there is no in-callback
+                // validator (e.g. TlsSession owns validation) and the options
+                // asked us to defer, try SSL_set_retry_verify so the handshake
+                // pauses here on OpenSSL 3.0+. If the runtime is older (1.1.x),
+                // accept the cert and let the caller validate after the
+                // handshake completes.
+                if (options.RemoteCertificateValidator is null && options.DeferCertificateValidation)
+                {
+                    if (Ssl.SslSetRetryVerify(sslHandle!) == 1)
+                    {
+                        return -1;
+                    }
+
+                    Ssl.X509StoreCtxSetError(storeCtx, (int)Interop.Crypto.X509VerifyStatusCodeUniversal.X509_V_OK);
+                    return 1;
+                }
 
                 // We need to note the number of certs in ExtraStore that were
                 // provided (by the user), we will add more from the received peer
@@ -888,7 +915,9 @@ internal static int CertVerifyCallback(IntPtr storeCtx, IntPtr arg)
                 try
                 {
                     ProtocolToken alertToken = default;
-                    if (options.SslStream!.VerifyRemoteCertificate(certificate, chain, options.CertificateContext?.Trust, ref alertToken, out SslPolicyErrors sslPolicyErrors, out X509ChainStatusFlags chainStatus))
+                    SslAuthenticationOptions.VerifyRemoteCertificateCallback? validator = options.RemoteCertificateValidator;
+                    Debug.Assert(validator is not null, "Expected SslAuthenticationOptions.RemoteCertificateValidator to be set by SslStream or TlsSession");
+                    if (validator!(certificate, chain, options.CertificateContext?.Trust, ref alertToken, out SslPolicyErrors sslPolicyErrors, out X509ChainStatusFlags chainStatus))
                     {
                         Ssl.X509StoreCtxSetError(storeCtx, (int)Interop.Crypto.X509VerifyStatusCodeUniversal.X509_V_OK);
                         return 1;

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Ssl.cs

@@ -190,6 +190,9 @@ internal static SafeSharedX509StackHandle SslGetPeerCertChain(SafeSslHandle ssl)
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslSetVerifyPeer")]
         internal static partial void SslSetVerifyPeer(SafeSslHandle ssl, [MarshalAs(UnmanagedType.Bool)] bool failIfNoPeerCert);
 
+        [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslSetRetryVerify")]
+        internal static partial int SslSetRetryVerify(SafeSslHandle ssl);
+
         [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslGetData")]
         internal static partial IntPtr SslGetData(IntPtr ssl);
 
@@ -381,6 +384,7 @@ internal enum SslErrorCode
             SSL_ERROR_WANT_X509_LOOKUP = 4,
             SSL_ERROR_SYSCALL = 5,
             SSL_ERROR_ZERO_RETURN = 6,
+            SSL_ERROR_WANT_RETRY_VERIFY = 12,
 
             // NOTE: this SslErrorCode value doesn't exist in OpenSSL, but
             // we use it to distinguish when a renegotiation is pending.
@@ -451,6 +455,10 @@ public static SafeSslHandle Create(SafeSslContextHandle context, SslAuthenticati
             handle._authOptionsHandle = new WeakGCHandle<SslAuthenticationOptions>(options);
             Interop.Ssl.SslSetData(handle, WeakGCHandle<SslAuthenticationOptions>.ToIntPtr(handle._authOptionsHandle));
 
+            // CertVerifyCallback needs the SafeSslHandle to stash a
+            // CertificateValidationException; expose it via the options.
+            options.SafeSslHandle = handle;
+
             // SslSetBio will transfer ownership of the BIO handles to the SSL context
             try
             {

src/libraries/System.Net.Security/ref/System.Net.Security.cs

@@ -687,6 +687,52 @@ public enum TlsCipherSuite : ushort
         TLS_ECDHE_PSK_WITH_AES_128_CCM_8_SHA256 = (ushort)53251,
         TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256 = (ushort)53253,
     }
+    public enum TlsOperationStatus
+    {
+        Complete = 0,
+        WantRead = 1,
+        WantWrite = 2,
+        Closed = 3,
+        WantCredentials = 4,
+        NeedsCertificateValidation = 5,
+        NeedsServerOptions = 6,
+    }
+    public sealed partial class TlsContext : System.IDisposable
+    {
+        internal TlsContext() { }
+        public bool IsServer { get { throw null; } }
+        public static System.Net.Security.TlsContext Create(System.Net.Security.SslServerAuthenticationOptions? options) { throw null; }
+        public static System.Net.Security.TlsContext Create(System.Net.Security.SslClientAuthenticationOptions options) { throw null; }
+        public void Dispose() { }
+    }
+    public sealed partial class TlsSession : System.IDisposable
+    {
+        internal TlsSession() { }
+        public bool IsServer { get { throw null; } }
+        public bool IsHandshakeComplete { get { throw null; } }
+        public bool HasPendingOutput { get { throw null; } }
+        public string? TargetHostName { get { throw null; } set { } }
+        public System.Net.Security.SslClientHelloInfo? ClientHelloInfo { get { throw null; } }
+        public System.Security.Authentication.SslProtocols NegotiatedProtocol { get { throw null; } }
+        [System.CLSCompliantAttribute(false)]
+        public System.Net.Security.TlsCipherSuite NegotiatedCipherSuite { get { throw null; } }
+        public System.Net.Security.SslApplicationProtocol NegotiatedApplicationProtocol { get { throw null; } }
+        public static System.Net.Security.TlsSession Create(System.Net.Security.TlsContext context) { throw null; }
+        public System.Net.Security.TlsOperationStatus ProcessHandshake(System.ReadOnlySpan<byte> input, System.Span<byte> output, out int consumed, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus Encrypt(System.ReadOnlySpan<byte> plaintext, System.Span<byte> ciphertext, out int consumed, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus Decrypt(System.ReadOnlySpan<byte> ciphertext, System.Span<byte> plaintext, out int consumed, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus Shutdown(System.Span<byte> ciphertext, out int produced) { throw null; }
+        public System.Net.Security.TlsOperationStatus DrainPendingOutput(System.Span<byte> ciphertext, out int produced) { throw null; }
+        public System.Security.Cryptography.X509Certificates.X509Certificate2? GetRemoteCertificate() { throw null; }
+        public System.Security.Cryptography.X509Certificates.X509Certificate2Collection? GetRemoteCertificates() { throw null; }
+        public System.Net.Security.SslPolicyErrors AcceptWithDefaultValidation() { throw null; }
+        public void SetRemoteCertificateValidationResult(System.Net.Security.SslPolicyErrors errors) { }
+        public void SetServerOptions(System.Net.Security.SslServerAuthenticationOptions options) { }
+        public System.Security.Cryptography.X509Certificates.X509Certificate2? LocalCertificate { get { throw null; } }
+        public System.Security.Authentication.ExtendedProtection.ChannelBinding? GetChannelBinding(System.Security.Authentication.ExtendedProtection.ChannelBindingKind kind) { throw null; }
+        public System.Net.Security.TlsOperationStatus RequestClientCertificate(System.Span<byte> ciphertext, out int produced) { throw null; }
+        public void Dispose() { }
+    }
 }
 namespace System.Security.Authentication
 {

src/libraries/System.Net.Security/src/System.Net.Security.csproj

@@ -62,6 +62,16 @@
     <Compile Include="System\Net\Security\SslStream.cs" />
     <Compile Include="System\Net\Security\SslStream.IO.cs" />
     <Compile Include="System\Net\Security\SslStream.Protocol.cs" />
+    <Compile Include="System\Net\Security\SslStream.TlsSessionWedge.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'linux' or '$(TargetPlatformIdentifier)' == 'freebsd' or '$(TargetPlatformIdentifier)' == 'windows'" />
+    <Compile Include="System\Net\Security\SslStream.NotUnix.cs"
+             Condition="'$(TargetPlatformIdentifier)' != 'linux' and '$(TargetPlatformIdentifier)' != 'freebsd' and '$(TargetPlatformIdentifier)' != 'windows'" />
+    <Compile Include="System\Net\Security\TlsContext.cs" />
+    <Compile Include="System\Net\Security\TlsOperationStatus.cs" />
+    <Compile Include="System\Net\Security\TlsSession.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'linux' or '$(TargetPlatformIdentifier)' == 'freebsd' or '$(TargetPlatformIdentifier)' == 'windows'" />
+    <Compile Include="System\Net\Security\TlsSession.Stub.cs"
+             Condition="'$(TargetPlatformIdentifier)' != 'linux' and '$(TargetPlatformIdentifier)' != 'freebsd' and '$(TargetPlatformIdentifier)' != 'windows'" />
     <Compile Include="System\Net\Security\SslStreamCertificateContext.cs" />
     <Compile Include="System\Net\Security\SslConnectionInfo.cs" />
     <Compile Include="System\Net\Security\StreamSizes.cs" />

src/libraries/System.Net.Security/src/System/Net/Security/NetEventSource.Security.cs

@@ -208,32 +208,32 @@ public void SspiSelectedCipherSuite(
 #pragma warning restore SYSLIB0058 // Use NegotiatedCipherSuite.
 
         [NonEvent]
-        public void RemoteCertificateError(SslStream SslStream, string message) =>
-            RemoteCertificateError(GetHashCode(SslStream), message);
+        public void RemoteCertificateError(object sender, string message) =>
+            RemoteCertificateError(GetHashCode(sender), message);
 
         [Event(RemoteCertificateErrorId, Level = EventLevel.Verbose)]
         private void RemoteCertificateError(int sslStreamHash, string message) =>
             WriteEvent(RemoteCertificateErrorId, sslStreamHash, message);
 
         [NonEvent]
-        public void RemoteCertDeclaredValid(SslStream SslStream) =>
-            RemoteCertDeclaredValid(GetHashCode(SslStream));
+        public void RemoteCertDeclaredValid(object sender) =>
+            RemoteCertDeclaredValid(GetHashCode(sender));
 
         [Event(RemoteVertificateValidId, Level = EventLevel.Verbose)]
         private void RemoteCertDeclaredValid(int sslStreamHash) =>
             WriteEvent(RemoteVertificateValidId, sslStreamHash);
 
         [NonEvent]
-        public void RemoteCertHasNoErrors(SslStream SslStream) =>
-            RemoteCertHasNoErrors(GetHashCode(SslStream));
+        public void RemoteCertHasNoErrors(object sender) =>
+            RemoteCertHasNoErrors(GetHashCode(sender));
 
         [Event(RemoteCertificateSuccessId, Level = EventLevel.Verbose)]
         private void RemoteCertHasNoErrors(int sslStreamHash) =>
             WriteEvent(RemoteCertificateSuccessId, sslStreamHash);
 
         [NonEvent]
-        public void RemoteCertUserDeclaredInvalid(SslStream SslStream) =>
-            RemoteCertUserDeclaredInvalid(GetHashCode(SslStream));
+        public void RemoteCertUserDeclaredInvalid(object sender) =>
+            RemoteCertUserDeclaredInvalid(GetHashCode(sender));
 
         [Event(RemoteCertificateInvalidId, Level = EventLevel.Verbose)]
         private void RemoteCertUserDeclaredInvalid(int sslStreamHash) =>

src/libraries/System.Net.Security/src/System/Net/Security/SslAuthenticationOptions.cs

@@ -225,6 +225,31 @@ internal void SetCertificateContextFromCert(X509Certificate2 certificate, bool?
 
 #if !TARGET_WINDOWS && !SYSNETSECURITY_NO_OPENSSL
         internal SslStream? SslStream { get; set; }
+
+        // Set by SafeSslHandle.Create so OpenSSL's CertVerifyCallback can stash
+        // a CertificateValidationException on the handle when validation fails.
+        // Typed as the base SafeHandle so this file compiles in test projects
+        // that don't include the OpenSSL interop sources.
+        internal System.Runtime.InteropServices.SafeHandle? SafeSslHandle { get; set; }
+
+        // Hook invoked by OpenSSL's CertVerifyCallback to drive remote
+        // certificate validation. Set by SslStream and by standalone TlsSession
+        // so both flows share the same callback plumbing.
+        internal delegate bool VerifyRemoteCertificateCallback(
+            X509Certificate2? certificate,
+            X509Chain? chain,
+            SslCertificateTrust? trust,
+            ref ProtocolToken alertToken,
+            out SslPolicyErrors sslPolicyErrors,
+            out X509ChainStatusFlags chainStatus);
+
+        internal VerifyRemoteCertificateCallback? RemoteCertificateValidator { get; set; }
+
+        // When true, the OpenSSL CertVerifyCallback always defers certificate
+        // validation. On OpenSSL 3.0+ the callback uses SSL_set_retry_verify
+        // to suspend the handshake; on older versions it accepts the cert and
+        // validation runs after the handshake completes. Set by TlsSession.
+        internal bool DeferCertificateValidation { get; set; }
 #endif
 
         public void Dispose()

src/libraries/System.Net.Security/src/System/Net/Security/SslSessionsCache.cs

@@ -114,15 +114,14 @@ public bool Equals(SslCredKey other)
             bool allowRsaPssPadding,
             bool allowRsaPkcs1Padding)
         {
+            var key = new SslCredKey(thumbPrint, (int)sslProtocols, isServer, encryptionPolicy, sendTrustList, checkRevocation, allowTlsResume, allowRsaPssPadding, allowRsaPkcs1Padding);
+
             if (s_cachedCreds.IsEmpty)
             {
                 if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(null, $"Not found, Current Cache Count = {s_cachedCreds.Count}");
                 return null;
             }
 
-            var key = new SslCredKey(thumbPrint, (int)sslProtocols, isServer, encryptionPolicy, sendTrustList, checkRevocation, allowTlsResume, allowRsaPssPadding, allowRsaPkcs1Padding);
-
-            //SafeCredentialReference? cached;
             SafeFreeCredentials? credentials = GetCachedCredential(key);
             if (credentials == null || credentials.IsClosed || credentials.IsInvalid || credentials.Expiry < DateTime.UtcNow)
             {

src/libraries/System.Net.Security/src/System/Net/Security/SslStream.NotUnix.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Net.Security
+{
+    // Stub partial impls for non-Linux/FreeBSD platforms. Always returns false so
+    // SslStream's existing PAL paths run unchanged.
+    public partial class SslStream
+    {
+#pragma warning disable CA1822 // partial method signature must match the Unix impl which is non-static
+        private partial bool TryNextMessageViaTlsSession(ReadOnlySpan<byte> incomingBuffer, out ProtocolToken token, out int consumed)
+        {
+            token = default;
+            consumed = 0;
+            return false;
+        }
+#pragma warning restore CA1822
+    }
+}