Brad Arant
2676aaf84c
A TLS sendData() failing with SSL_ERROR_WANT_READ (mid-handshake application data, or a post-handshake session ticket exchange) has nothing to do with write-readiness, but resetSocket() kept re-arming EPOLLWRNORM anyway - an already-writable TCP socket fires that immediately, so the retry loop spun the CPU as fast as it could until the connection was eventually dropped, without ever reading the peer's first command. Confirmed via a 15GB+ debug log from one stuck connection. sendNeedsRead() lets Socket now tell the difference and wait for a real EPOLLIN instead of busy-retrying.
283 lines
12 KiB
C++
283 lines
12 KiB
C++
#include "TLS.h"
|
|
#include "Exception.h"
|
|
#include "Log.h"
|
|
#include <iostream>
|
|
#include <openssl/rand.h>
|
|
|
|
namespace core {
|
|
|
|
static pthread_mutex_t *lockarray;
|
|
|
|
static unsigned long thread_id(void) {
|
|
return ((unsigned long) pthread_self());
|
|
}
|
|
|
|
static void lock_callback(int mode, int type, const char *file, int line) {
|
|
if(mode & CRYPTO_LOCK)
|
|
pthread_mutex_lock(&(lockarray[type]));
|
|
else
|
|
pthread_mutex_unlock(&(lockarray[type]));
|
|
}
|
|
|
|
static int generate_session_id(const SSL *ssl, unsigned char *id, unsigned int *id_len) {
|
|
char *session_id_prefix = (char *)"BARANT";
|
|
unsigned int count = 0;
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Generating unique session id.";
|
|
do {
|
|
RAND_bytes(id, *id_len);
|
|
memcpy(id, session_id_prefix, (strlen(session_id_prefix) < *id_len));
|
|
} while(SSL_has_matching_session_id(ssl, id, *id_len) && (++count < 10));
|
|
return 1;
|
|
}
|
|
|
|
void handshake_complete(const SSL *ssl, int where, int ret) {
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "==>" << SSL_state_string_long(ssl) << "<==" << ret;
|
|
if(where & SSL_CB_HANDSHAKE_DONE) {
|
|
// Only a context configured with SSL_VERIFY_PEER (see
|
|
// createContext()'s own cACertificate-gated block) ever sends a
|
|
// client a CertificateRequest in the first place - ordinary
|
|
// one-way TLS (a public site's normal case, no caCertificate
|
|
// configured) never asks the client for one, so
|
|
// SSL_get_peer_certificate() is unconditionally NULL there and is
|
|
// not a failure. Checking the mode actually negotiated on THIS
|
|
// connection (inherited from the context) rather than assuming
|
|
// every handshake is mutual-TLS is what makes plain
|
|
// browser-facing HTTPS work at all under this callback.
|
|
if(SSL_get_verify_mode(ssl) & SSL_VERIFY_PEER) {
|
|
X509 *ssl_client_cert = SSL_get_peer_certificate(ssl);
|
|
if(!ssl_client_cert)
|
|
throw std::string("Unable to get peer certificate.");
|
|
X509_free(ssl_client_cert);
|
|
if(SSL_get_verify_result(ssl) != X509_V_OK)
|
|
throw std::string("Certificate verification failed.");
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Certificate verified successfully.";
|
|
}
|
|
}
|
|
else
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "No client certificate.";
|
|
}
|
|
|
|
TLS::TLS() {}
|
|
|
|
TLS::TLS(TLSInfo *tlsInfo) {
|
|
createContext(tlsInfo);
|
|
}
|
|
|
|
TLS::TLS(SSL_CTX *ctx) : ctx(ctx) {}
|
|
|
|
TLS::~TLS() {}
|
|
|
|
void TLS::createContext(TLSInfo *tlsInfo) {
|
|
|
|
if(tlsInfo) {
|
|
SSL_library_init();
|
|
SSL_load_error_strings();
|
|
lockarray = (pthread_mutex_t *)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t));
|
|
for(int i = 0; i < CRYPTO_num_locks(); ++i)
|
|
pthread_mutex_init(&(lockarray[i]), NULL);
|
|
CRYPTO_set_id_callback((unsigned long (*)())thread_id);
|
|
CRYPTO_set_locking_callback((void (*)(int, int, const char *, int))lock_callback);
|
|
SSLeay_add_ssl_algorithms();
|
|
RAND_load_file("/dev/hwrng", 1024);
|
|
if(!(ctx = SSL_CTX_new(SSLv23_server_method())))
|
|
throw coreutils::Exception("Error while setting server method SSLv23.");
|
|
SSL_CTX_set_mode(ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
|
|
SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET);
|
|
SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER);
|
|
// SSL_CTX_set_generate_session_id(ctx, generate_session_id);
|
|
SSL_CTX_set_cipher_list(ctx, "ECDH-ECDSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA");
|
|
// _chain_file (not _file) - loads every certificate in the file,
|
|
// not just the first. A real Let's Encrypt fullchain.pem is the
|
|
// leaf cert followed by the intermediate CA cert concatenated in
|
|
// one file specifically so a server can hand both to the client
|
|
// in one shot; SSL_CTX_use_certificate_file() only ever reads the
|
|
// first PEM block, silently dropping the intermediate - confirmed
|
|
// via a real handshake against this exact fullchain.pem: clients
|
|
// got "unable to get local issuer certificate" because the server
|
|
// only ever sent the leaf, never the intermediate needed to chain
|
|
// up to a root the client already trusts.
|
|
if(SSL_CTX_use_certificate_chain_file(ctx, tlsInfo->certificate.c_str()) <= 0)
|
|
throw coreutils::Exception("Error looking up certificate.");
|
|
if(SSL_CTX_use_PrivateKey_file(ctx, tlsInfo->key.c_str(), SSL_FILETYPE_PEM) < 0)
|
|
throw coreutils::Exception("Error with private key.");
|
|
if(SSL_CTX_check_private_key(ctx) != 1)
|
|
throw coreutils::Exception("Private key does not match certificate.");
|
|
// Client-certificate verification (mutual TLS) is opt-in, not the
|
|
// default - a public-facing site (e.g. a real Let's Encrypt
|
|
// server cert with no corresponding client CA of its own) has no
|
|
// client certificate to hand out to ordinary visitors, so
|
|
// requesting one via SSL_VERIFY_PEER would surface a "select a
|
|
// certificate" prompt real browsers can't meaningfully answer.
|
|
// Only enable it when a caCertificate is actually configured -
|
|
// existing internal/mTLS use of this class (any config that DOES
|
|
// set tls.caCertificate) keeps behaving exactly as before.
|
|
if(tlsInfo->cACertificate.getLength() > 0) {
|
|
SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL);
|
|
SSL_CTX_set_verify_depth(ctx, 1);
|
|
if(!SSL_CTX_load_verify_locations(ctx, tlsInfo->cACertificate.c_str(), NULL))
|
|
throw coreutils::Exception("Cannot verify locations.");
|
|
SSL_CTX_set_client_CA_list(ctx, SSL_load_client_CA_file(tlsInfo->cACertificate.c_str()));
|
|
coreutils::Log(coreutils::LOG_INFO) << "Server key authenticated (client certificate required).";
|
|
} else {
|
|
coreutils::Log(coreutils::LOG_INFO) << "Server key authenticated (no client certificate required).";
|
|
}
|
|
}
|
|
}
|
|
|
|
void TLS::registerSocket(int fd) {
|
|
int ret;
|
|
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "TLS socket initializing on socket " << fd << "...";
|
|
|
|
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
|
|
|
|
ssl = SSL_new(ctx);
|
|
// if(ssl <= 0)
|
|
// throw std::string("Error creating new TLS socket.");
|
|
|
|
SSL_set_info_callback(ssl, handshake_complete);
|
|
|
|
if((ret = SSL_set_fd(ssl, fd)) == 0)
|
|
throw std::string("Error setting TLS socket descriptor.");
|
|
|
|
// if(!SSL_set_generate_session_id(ssl, generate_session_id))
|
|
// throw std::string("Error setting session identifier callback.");
|
|
|
|
// Was set unconditionally as the FIRST statement in this
|
|
// function (before `ssl` was even assigned) - a flag meant to
|
|
// mean "ssl is a valid, ready-to-use SSL*" needs to become true
|
|
// only once that's actually true, at the end, not the start.
|
|
// Previously set but never actually READ anywhere either -
|
|
// sendData()/receiveData() below now check it before touching
|
|
// `ssl` at all. See their own doc comments for the real,
|
|
// confirmed crash this fixes: a brand-new TLS session's own
|
|
// greeting (queued in onConnected(), see e.g. IMAPSession.h)
|
|
// gets written via Socket::write() -> resetSocket(), which - deep
|
|
// inside TCPServer::accept(), BEFORE TCPServer::connectionRequest()
|
|
// ever calls this session's own performTLSHandshake() - makes the
|
|
// fd immediately eligible for an EPOLLWRNORM event on a SEPARATE
|
|
// epoll worker thread. That thread's TCPSocket::sendData() takes
|
|
// ioLock (see its own doc comment) and calls straight into
|
|
// SSL_write() on `ssl`, which is genuinely uninitialized garbage
|
|
// (TLS.h explicitly documents `ssl` has no default member
|
|
// initializer) until THIS function runs - ioLock alone doesn't
|
|
// prevent it, since performTLSHandshake() (the only other place
|
|
// that takes ioLock) hasn't even been called yet at that point.
|
|
// Confirmed live via gdb under a mixed-protocol connection-churn
|
|
// stress test: SIGSEGV inside libssl.so's SSL_write(), called from
|
|
// TCPSocket::sendData() with ioGuard already held, while
|
|
// registerSocket()/acceptSocket() for that same session were still
|
|
// pending on the accepting thread.
|
|
initialized = true;
|
|
}
|
|
|
|
void TLS::acceptSocket() {
|
|
|
|
switch (SSL_get_error(ssl, SSL_accept(ssl))) {
|
|
case SSL_ERROR_SSL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SSL on ssl_accept. errno=" << errno;
|
|
break;
|
|
case SSL_ERROR_WANT_READ:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_READ on ssl_accept.";
|
|
break;
|
|
case SSL_ERROR_WANT_WRITE:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_WRITE on ssl_accept.";
|
|
break;
|
|
case SSL_ERROR_SYSCALL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SYSCALL on ssl_accept. errno=" << errno;
|
|
// shutdown();
|
|
break;
|
|
default:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Unknown ERROR on ssl_accept.";
|
|
break;
|
|
}
|
|
|
|
}
|
|
|
|
int TLS::receiveData(coreutils::ZString &buffer) {
|
|
|
|
// `ssl` is genuinely uninitialized until registerSocket() has
|
|
// fully run (see its own doc comment on `initialized` for the
|
|
// real crash this prevents) - a racing thread landing here first
|
|
// just gets "nothing yet", exactly like a real SSL_ERROR_WANT_READ
|
|
// would report, rather than touching a garbage pointer.
|
|
if (!initialized)
|
|
return -1;
|
|
|
|
int len;
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "receiveData TLS on socket.";
|
|
|
|
if((len = ::SSL_read(ssl, buffer.getData(), buffer.getLength())) >= 0) {
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "receiveData TLS...len=" << len;
|
|
return len;
|
|
}
|
|
else {
|
|
switch (SSL_get_error(ssl, len)) {
|
|
case SSL_ERROR_SSL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SSL on ssl_read. error=" << errno;
|
|
break;
|
|
case SSL_ERROR_WANT_READ:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_READ on ssl_read.";
|
|
break;
|
|
case SSL_ERROR_WANT_WRITE:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_WRITE on ssl_read.";
|
|
break;
|
|
case SSL_ERROR_SYSCALL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SYSCALL on ssl_read. errno=" << errno;
|
|
break;
|
|
default:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Unknown ERROR on ssl_read.";
|
|
break;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
}
|
|
|
|
int TLS::sendData(const char *data, int length) {
|
|
|
|
// Same "ssl isn't ready yet" guard as receiveData() - see
|
|
// registerSocket()'s doc comment on `initialized` for the real,
|
|
// confirmed crash this prevents (SIGSEGV in SSL_write() on a
|
|
// still-uninitialized `ssl`, reached via a brand-new session's own
|
|
// greeting racing ahead of its own TLS handshake). Returning <= 0
|
|
// here is already the exact case Socket::writeSocket() treats as
|
|
// "nothing sent this round, leave it queued and retry on the next
|
|
// EPOLLWRNORM" - no special-casing needed at any call site.
|
|
if (!initialized)
|
|
return -1;
|
|
|
|
int len;
|
|
|
|
if((len = ::SSL_write(ssl, data, length)) >= 0) {
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "sendData TLS...len=" << len;
|
|
lastSendWantedRead = false;
|
|
return len;
|
|
}
|
|
else {
|
|
lastSendWantedRead = false;
|
|
switch (SSL_get_error(ssl, len)) {
|
|
case SSL_ERROR_SSL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SSL on ssl_write. error=" << errno;
|
|
break;
|
|
case SSL_ERROR_WANT_READ:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_READ on ssl_write.";
|
|
lastSendWantedRead = true;
|
|
break;
|
|
case SSL_ERROR_WANT_WRITE:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_WRITE on ssl_write.";
|
|
break;
|
|
case SSL_ERROR_SYSCALL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SYSCALL on ssl_write. errno=" << errno;
|
|
break;
|
|
default:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Unknown ERROR on ssl_write.";
|
|
break;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|