Brad ArantandClaude Sonnet 5
1d3d30cd05
- INotify/Thread epoll dispatch was silently dropping every live core::INotify event (an isKnownINotify registry fixes it) - this had broken mail delivery for over a week, masked by routine restarts. - TCPServer::lock self-deadlocked every admin-console connection via a plain std::mutex being re-entered; switched to recursive_mutex. - TLS.cpp/h: SNI-based certificate selection (addSNIContext), the groundwork for multi-tenant TLS (qualityiron.vegas alongside barant.com on one APIServer process). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
322 lines
13 KiB
C++
322 lines
13 KiB
C++
#include "TLS.h"
|
|
#include "Exception.h"
|
|
#include "Log.h"
|
|
#include <iostream>
|
|
#include <openssl/rand.h>
|
|
|
|
namespace core {
|
|
|
|
static pthread_mutex_t *lockarray;
|
|
|
|
static unsigned long thread_id(void) {
|
|
return ((unsigned long) pthread_self());
|
|
}
|
|
|
|
static void lock_callback(int mode, int type, const char *file, int line) {
|
|
if(mode & CRYPTO_LOCK)
|
|
pthread_mutex_lock(&(lockarray[type]));
|
|
else
|
|
pthread_mutex_unlock(&(lockarray[type]));
|
|
}
|
|
|
|
static int generate_session_id(const SSL *ssl, unsigned char *id, unsigned int *id_len) {
|
|
char *session_id_prefix = (char *)"BARANT";
|
|
unsigned int count = 0;
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Generating unique session id.";
|
|
do {
|
|
RAND_bytes(id, *id_len);
|
|
memcpy(id, session_id_prefix, (strlen(session_id_prefix) < *id_len));
|
|
} while(SSL_has_matching_session_id(ssl, id, *id_len) && (++count < 10));
|
|
return 1;
|
|
}
|
|
|
|
void handshake_complete(const SSL *ssl, int where, int ret) {
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "==>" << SSL_state_string_long(ssl) << "<==" << ret;
|
|
if(where & SSL_CB_HANDSHAKE_DONE) {
|
|
// Only a context configured with SSL_VERIFY_PEER (see
|
|
// createContext()'s own cACertificate-gated block) ever sends a
|
|
// client a CertificateRequest in the first place - ordinary
|
|
// one-way TLS (a public site's normal case, no caCertificate
|
|
// configured) never asks the client for one, so
|
|
// SSL_get_peer_certificate() is unconditionally NULL there and is
|
|
// not a failure. Checking the mode actually negotiated on THIS
|
|
// connection (inherited from the context) rather than assuming
|
|
// every handshake is mutual-TLS is what makes plain
|
|
// browser-facing HTTPS work at all under this callback.
|
|
if(SSL_get_verify_mode(ssl) & SSL_VERIFY_PEER) {
|
|
X509 *ssl_client_cert = SSL_get_peer_certificate(ssl);
|
|
if(!ssl_client_cert)
|
|
throw std::string("Unable to get peer certificate.");
|
|
X509_free(ssl_client_cert);
|
|
if(SSL_get_verify_result(ssl) != X509_V_OK)
|
|
throw std::string("Certificate verification failed.");
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Certificate verified successfully.";
|
|
}
|
|
}
|
|
else
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "No client certificate.";
|
|
}
|
|
|
|
TLS::TLS() {}
|
|
|
|
TLS::TLS(TLSInfo *tlsInfo) {
|
|
createContext(tlsInfo);
|
|
}
|
|
|
|
TLS::TLS(SSL_CTX *ctx) : ctx(ctx) {}
|
|
|
|
TLS::~TLS() {}
|
|
|
|
void TLS::createContext(TLSInfo *tlsInfo) {
|
|
|
|
if(tlsInfo) {
|
|
SSL_library_init();
|
|
SSL_load_error_strings();
|
|
lockarray = (pthread_mutex_t *)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t));
|
|
for(int i = 0; i < CRYPTO_num_locks(); ++i)
|
|
pthread_mutex_init(&(lockarray[i]), NULL);
|
|
CRYPTO_set_id_callback((unsigned long (*)())thread_id);
|
|
CRYPTO_set_locking_callback((void (*)(int, int, const char *, int))lock_callback);
|
|
SSLeay_add_ssl_algorithms();
|
|
RAND_load_file("/dev/hwrng", 1024);
|
|
ctx = buildContext(tlsInfo);
|
|
}
|
|
}
|
|
|
|
// The per-context cert/key/cipher/verify setup factored out of
|
|
// createContext() above so addSNIContext() (below) can build
|
|
// additional contexts - one per extra hostname - without repeating
|
|
// the ONE-TIME, process-wide OpenSSL library init that stays in
|
|
// createContext() itself (SSL_library_init(), the lockarray, RAND
|
|
// seeding): that part must only ever run once per process, not once
|
|
// per hostname.
|
|
SSL_CTX *TLS::buildContext(TLSInfo *tlsInfo) {
|
|
SSL_CTX *newCtx;
|
|
if(!(newCtx = SSL_CTX_new(SSLv23_server_method())))
|
|
throw coreutils::Exception("Error while setting server method SSLv23.");
|
|
SSL_CTX_set_mode(newCtx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
|
|
SSL_CTX_set_options(newCtx, SSL_OP_NO_TICKET);
|
|
SSL_CTX_set_session_cache_mode(newCtx, SSL_SESS_CACHE_SERVER);
|
|
SSL_CTX_set_cipher_list(newCtx, "ECDH-ECDSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA");
|
|
// _chain_file (not _file) - loads every certificate in the file,
|
|
// not just the first. A real Let's Encrypt fullchain.pem is the
|
|
// leaf cert followed by the intermediate CA cert concatenated in
|
|
// one file specifically so a server can hand both to the client
|
|
// in one shot; SSL_CTX_use_certificate_file() only ever reads the
|
|
// first PEM block, silently dropping the intermediate - confirmed
|
|
// via a real handshake against this exact fullchain.pem: clients
|
|
// got "unable to get local issuer certificate" because the server
|
|
// only ever sent the leaf, never the intermediate needed to chain
|
|
// up to a root the client already trusts.
|
|
if(SSL_CTX_use_certificate_chain_file(newCtx, tlsInfo->certificate.c_str()) <= 0)
|
|
throw coreutils::Exception("Error looking up certificate.");
|
|
if(SSL_CTX_use_PrivateKey_file(newCtx, tlsInfo->key.c_str(), SSL_FILETYPE_PEM) < 0)
|
|
throw coreutils::Exception("Error with private key.");
|
|
if(SSL_CTX_check_private_key(newCtx) != 1)
|
|
throw coreutils::Exception("Private key does not match certificate.");
|
|
// Client-certificate verification (mutual TLS) is opt-in, not the
|
|
// default - a public-facing site (e.g. a real Let's Encrypt
|
|
// server cert with no corresponding client CA of its own) has no
|
|
// client certificate to hand out to ordinary visitors, so
|
|
// requesting one via SSL_VERIFY_PEER would surface a "select a
|
|
// certificate" prompt real browsers can't meaningfully answer.
|
|
// Only enable it when a caCertificate is actually configured -
|
|
// existing internal/mTLS use of this class (any config that DOES
|
|
// set tls.caCertificate) keeps behaving exactly as before.
|
|
if(tlsInfo->cACertificate.getLength() > 0) {
|
|
SSL_CTX_set_verify(newCtx, SSL_VERIFY_PEER, NULL);
|
|
SSL_CTX_set_verify_depth(newCtx, 1);
|
|
if(!SSL_CTX_load_verify_locations(newCtx, tlsInfo->cACertificate.c_str(), NULL))
|
|
throw coreutils::Exception("Cannot verify locations.");
|
|
SSL_CTX_set_client_CA_list(newCtx, SSL_load_client_CA_file(tlsInfo->cACertificate.c_str()));
|
|
coreutils::Log(coreutils::LOG_INFO) << "Server key authenticated (client certificate required).";
|
|
} else {
|
|
coreutils::Log(coreutils::LOG_INFO) << "Server key authenticated (no client certificate required).";
|
|
}
|
|
return newCtx;
|
|
}
|
|
|
|
int TLS::sniCallback(SSL *ssl, int *ad, void *arg) {
|
|
TLS *self = (TLS *)arg;
|
|
const char *name = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);
|
|
if(name) {
|
|
auto it = self->sniContexts.find(name);
|
|
if(it != self->sniContexts.end())
|
|
SSL_set_SSL_CTX(ssl, it->second);
|
|
// No match (an unconfigured hostname, or a typo'd Host that still
|
|
// resolves here via DNS) - fall through and keep whatever SSL_CTX
|
|
// this SSL* already has, which is the base `ctx` SSL_new() was
|
|
// constructed from. Exactly the same cert a pre-SNI client (or
|
|
// one sending no SNI at all) already gets today.
|
|
}
|
|
return SSL_TLSEXT_ERR_OK;
|
|
}
|
|
|
|
void TLS::addSNIContext(const std::string &hostname, TLSInfo *tlsInfo) {
|
|
SSL_CTX *newCtx = buildContext(tlsInfo);
|
|
sniContexts[hostname] = newCtx;
|
|
if(!sniCallbackRegistered) {
|
|
SSL_CTX_set_tlsext_servername_callback(ctx, sniCallback);
|
|
SSL_CTX_set_tlsext_servername_arg(ctx, this);
|
|
sniCallbackRegistered = true;
|
|
}
|
|
}
|
|
|
|
|
|
void TLS::registerSocket(int fd) {
|
|
int ret;
|
|
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "TLS socket initializing on socket " << fd << "...";
|
|
|
|
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
|
|
|
|
ssl = SSL_new(ctx);
|
|
// if(ssl <= 0)
|
|
// throw std::string("Error creating new TLS socket.");
|
|
|
|
SSL_set_info_callback(ssl, handshake_complete);
|
|
|
|
if((ret = SSL_set_fd(ssl, fd)) == 0)
|
|
throw std::string("Error setting TLS socket descriptor.");
|
|
|
|
// if(!SSL_set_generate_session_id(ssl, generate_session_id))
|
|
// throw std::string("Error setting session identifier callback.");
|
|
|
|
// Was set unconditionally as the FIRST statement in this
|
|
// function (before `ssl` was even assigned) - a flag meant to
|
|
// mean "ssl is a valid, ready-to-use SSL*" needs to become true
|
|
// only once that's actually true, at the end, not the start.
|
|
// Previously set but never actually READ anywhere either -
|
|
// sendData()/receiveData() below now check it before touching
|
|
// `ssl` at all. See their own doc comments for the real,
|
|
// confirmed crash this fixes: a brand-new TLS session's own
|
|
// greeting (queued in onConnected(), see e.g. IMAPSession.h)
|
|
// gets written via Socket::write() -> resetSocket(), which - deep
|
|
// inside TCPServer::accept(), BEFORE TCPServer::connectionRequest()
|
|
// ever calls this session's own performTLSHandshake() - makes the
|
|
// fd immediately eligible for an EPOLLWRNORM event on a SEPARATE
|
|
// epoll worker thread. That thread's TCPSocket::sendData() takes
|
|
// ioLock (see its own doc comment) and calls straight into
|
|
// SSL_write() on `ssl`, which is genuinely uninitialized garbage
|
|
// (TLS.h explicitly documents `ssl` has no default member
|
|
// initializer) until THIS function runs - ioLock alone doesn't
|
|
// prevent it, since performTLSHandshake() (the only other place
|
|
// that takes ioLock) hasn't even been called yet at that point.
|
|
// Confirmed live via gdb under a mixed-protocol connection-churn
|
|
// stress test: SIGSEGV inside libssl.so's SSL_write(), called from
|
|
// TCPSocket::sendData() with ioGuard already held, while
|
|
// registerSocket()/acceptSocket() for that same session were still
|
|
// pending on the accepting thread.
|
|
initialized = true;
|
|
}
|
|
|
|
void TLS::acceptSocket() {
|
|
|
|
switch (SSL_get_error(ssl, SSL_accept(ssl))) {
|
|
case SSL_ERROR_SSL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SSL on ssl_accept. errno=" << errno;
|
|
break;
|
|
case SSL_ERROR_WANT_READ:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_READ on ssl_accept.";
|
|
break;
|
|
case SSL_ERROR_WANT_WRITE:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_WRITE on ssl_accept.";
|
|
break;
|
|
case SSL_ERROR_SYSCALL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SYSCALL on ssl_accept. errno=" << errno;
|
|
// shutdown();
|
|
break;
|
|
default:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Unknown ERROR on ssl_accept.";
|
|
break;
|
|
}
|
|
|
|
}
|
|
|
|
int TLS::receiveData(coreutils::ZString &buffer) {
|
|
|
|
// `ssl` is genuinely uninitialized until registerSocket() has
|
|
// fully run (see its own doc comment on `initialized` for the
|
|
// real crash this prevents) - a racing thread landing here first
|
|
// just gets "nothing yet", exactly like a real SSL_ERROR_WANT_READ
|
|
// would report, rather than touching a garbage pointer.
|
|
if (!initialized)
|
|
return -1;
|
|
|
|
int len;
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "receiveData TLS on socket.";
|
|
|
|
if((len = ::SSL_read(ssl, buffer.getData(), buffer.getLength())) >= 0) {
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "receiveData TLS...len=" << len;
|
|
return len;
|
|
}
|
|
else {
|
|
switch (SSL_get_error(ssl, len)) {
|
|
case SSL_ERROR_SSL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SSL on ssl_read. error=" << errno;
|
|
break;
|
|
case SSL_ERROR_WANT_READ:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_READ on ssl_read.";
|
|
break;
|
|
case SSL_ERROR_WANT_WRITE:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_WRITE on ssl_read.";
|
|
break;
|
|
case SSL_ERROR_SYSCALL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SYSCALL on ssl_read. errno=" << errno;
|
|
break;
|
|
default:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Unknown ERROR on ssl_read.";
|
|
break;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
}
|
|
|
|
int TLS::sendData(const char *data, int length) {
|
|
|
|
// Same "ssl isn't ready yet" guard as receiveData() - see
|
|
// registerSocket()'s doc comment on `initialized` for the real,
|
|
// confirmed crash this prevents (SIGSEGV in SSL_write() on a
|
|
// still-uninitialized `ssl`, reached via a brand-new session's own
|
|
// greeting racing ahead of its own TLS handshake). Returning <= 0
|
|
// here is already the exact case Socket::writeSocket() treats as
|
|
// "nothing sent this round, leave it queued and retry on the next
|
|
// EPOLLWRNORM" - no special-casing needed at any call site.
|
|
if (!initialized)
|
|
return -1;
|
|
|
|
int len;
|
|
|
|
if((len = ::SSL_write(ssl, data, length)) >= 0) {
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "sendData TLS...len=" << len;
|
|
lastSendWantedRead = false;
|
|
return len;
|
|
}
|
|
else {
|
|
lastSendWantedRead = false;
|
|
switch (SSL_get_error(ssl, len)) {
|
|
case SSL_ERROR_SSL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SSL on ssl_write. error=" << errno;
|
|
break;
|
|
case SSL_ERROR_WANT_READ:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_READ on ssl_write.";
|
|
lastSendWantedRead = true;
|
|
break;
|
|
case SSL_ERROR_WANT_WRITE:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_WANT_WRITE on ssl_write.";
|
|
break;
|
|
case SSL_ERROR_SYSCALL:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "ERROR_SYSCALL on ssl_write. errno=" << errno;
|
|
break;
|
|
default:
|
|
coreutils::Log(coreutils::LOG_DEBUG_3) << "Unknown ERROR on ssl_write.";
|
|
break;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|