From ec07f3ea7c403a0378d8ab2e232fd5998686fb4d Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Thu, 6 Aug 2026 00:29:12 +0545 Subject: [PATCH 1/2] feat: resume TLS sessions instead of a full handshake per connection Every https connection started a full handshake, even against a peer this worker handshook with seconds earlier and that was offering to resume. On the connection-per-request TLS shape the handshake is most of the request cost. A per-worker cache holds one session per TLS peer, up to 256, evicting the least recently used and dropping expired ones on the way out. A fresh connection offers the session for its key before the handshake; OpenSSL reports new sessions through the session callback, which is what makes TLS 1.3 resume too, since its tickets arrive after the handshake rather than during it. The cache key is the verify mode, the peer address, the SNI and the trusted CA - every input the original handshake was judged on, since a resumed one skips the certificate exchange and answers with the verify result stored in the session. A session is published only once the handshake has passed the checks the caller asked for, so a rejected peer leaves nothing behind. Resumption is on by default and both entry points take ssl_session_reuse = false to turn it off, as nginx's proxy_ssl_session_reuse does. --- README.md | 26 ++ config | 1 + lib/resty/ngx_http_ffi_client.lua | 14 +- src/ngx_http_ffi_client.h | 25 ++ src/ngx_http_ffi_client_request.c | 19 + src/ngx_http_ffi_client_ssl_session.c | 515 ++++++++++++++++++++++++++ t/016-tls-session-resumption.t | 409 ++++++++++++++++++++ 7 files changed, 1008 insertions(+), 1 deletion(-) create mode 100644 src/ngx_http_ffi_client_ssl_session.c create mode 100644 t/016-tls-session-resumption.t diff --git a/README.md b/README.md index 6cd5b6e..8e36afa 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,32 @@ The one-shot `request_uri{ keepalive = { pool, pool_size, idle_timeout } }` resolves the same defaults and shares the same pools, so a pool name reaches the same connections from either entry point. +### TLS session resumption + +An https connection offers the server the session from an earlier handshake with +the same peer, so it costs one round trip instead of two and no certificate work. +Both entry points do it, and both take `ssl_session_reuse = false` to turn it +off, the way nginx's `proxy_ssl_session_reuse` does. It matters most where the +connection pool cannot help: a fresh connection per request, which is what the +`tlsshort` benchmark shape measures. + +- The cache holds one session per TLS peer and lives for the worker, up to `256` + peers, evicting the least recently used. A session past its expiry is dropped + rather than offered. +- The cache key is the verify mode, the peer address, the SNI and the trusted CA + — every input the original handshake was judged on. A resumed handshake skips + the certificate exchange and answers with the verify result recorded in the + session, so a session established with verification off is never offered to a + request that asked for it on, and one trusting a given CA is never offered to a + request naming another. +- Only a handshake this client accepted leaves a session behind. A peer rejected + for a bad chain or a name mismatch leaves nothing for the next request. +- TLS 1.3 resumes as well as TLS 1.2: its tickets arrive after the handshake and + are picked up from the session callback rather than at handshake completion. +- A resumed handshake does not re-send the peer certificate. Nothing in this + client reads it outside the verify path, which reads it from the session, but + it is the observable difference to know about. + ## Project status This project is at an **early implementation stage**. The protocol direction is diff --git a/config b/config index be3410f..35bc046 100644 --- a/config +++ b/config @@ -26,6 +26,7 @@ ngx_http_ffi_client_srcs="\ $ngx_addon_dir/src/ngx_http_ffi_client_module.c \ $ngx_addon_dir/src/ngx_http_ffi_client_lua.c \ $ngx_addon_dir/src/ngx_http_ffi_client_keepalive.c \ + $ngx_addon_dir/src/ngx_http_ffi_client_ssl_session.c \ $ngx_addon_dir/src/ngx_http_ffi_client_request.c" if [ "$ngx_http_ffi_client_use_llhttp" = "1" ]; then diff --git a/lib/resty/ngx_http_ffi_client.lua b/lib/resty/ngx_http_ffi_client.lua index 6c3963f..aa00eb7 100644 --- a/lib/resty/ngx_http_ffi_client.lua +++ b/lib/resty/ngx_http_ffi_client.lua @@ -60,6 +60,7 @@ if not pcall(ffi.typeof, "ngx_http_ffi_client_ffi_req_t") then uintptr_t ssl_verify; ngx_http_ffi_client_ffi_str_t ssl_server_name; ngx_http_ffi_client_ffi_str_t ssl_trusted_certificate; + uintptr_t ssl_session_reuse; } ngx_http_ffi_client_ffi_req_t; typedef struct { @@ -91,6 +92,7 @@ if not pcall(ffi.typeof, "ngx_http_ffi_client_ffi_req_t") then uintptr_t ssl_verify; ngx_http_ffi_client_ffi_str_t ssl_server_name; ngx_http_ffi_client_ffi_str_t ssl_trusted_certificate; + uintptr_t ssl_session_reuse; } ngx_http_ffi_client_ffi_connect_t; typedef struct { @@ -327,6 +329,9 @@ function _M.request_uri(opts) -- verify defaults on for https; pass ssl_verify = false to disable local ssl_verify = ssl and opts.ssl_verify ~= false + -- resumption defaults on; pass ssl_session_reuse = false to disable + local ssl_session_reuse = ssl and opts.ssl_session_reuse ~= false + if opts.ssl_server_name ~= nil and (type(opts.ssl_server_name) ~= "string" or opts.ssl_server_name == "") @@ -412,6 +417,7 @@ function _M.request_uri(opts) req.read_timeout = tonumber(opts.read_timeout or opts.timeout or 60000) req.ssl = ssl and 1 or 0 req.ssl_verify = ssl_verify and 1 or 0 + req.ssl_session_reuse = ssl_session_reuse and 1 or 0 local err = get_string_buf(ERR_BUF_SIZE) local errlen = get_size_ptr() @@ -543,7 +549,7 @@ end -- connect{ scheme, host, port, pool, pool_size, ssl_verify, ssl_server_name, --- ssl_trusted_certificate } or connect(host, port) +-- ssl_trusted_certificate, ssl_session_reuse } or connect(host, port) function client.connect(self, opts, port_arg) local r = get_request() if not r then @@ -552,6 +558,7 @@ function client.connect(self, opts, port_arg) local host, port, scheme, pool, pool_size local ssl_verify, ssl_server_name, ssl_trusted_certificate + local ssl_session_reuse if type(opts) == "table" then host = opts.host @@ -562,6 +569,7 @@ function client.connect(self, opts, port_arg) ssl_verify = opts.ssl_verify ssl_server_name = opts.ssl_server_name ssl_trusted_certificate = opts.ssl_trusted_certificate + ssl_session_reuse = opts.ssl_session_reuse else host = opts @@ -587,6 +595,9 @@ function client.connect(self, opts, port_arg) -- verify defaults on for https; pass ssl_verify = false to disable ssl_verify = ssl and ssl_verify ~= false + -- resumption defaults on; pass ssl_session_reuse = false to disable + ssl_session_reuse = ssl and ssl_session_reuse ~= false + if ssl_server_name ~= nil and (type(ssl_server_name) ~= "string" or ssl_server_name == "") then @@ -643,6 +654,7 @@ function client.connect(self, opts, port_arg) cp.keepalive_idle_timeout = 0 cp.ssl = ssl and 1 or 0 cp.ssl_verify = ssl_verify and 1 or 0 + cp.ssl_session_reuse = ssl_session_reuse and 1 or 0 local err, errlen = new_err_buf() local rc = c_connect(r, self._op, cp, err, errlen) diff --git a/src/ngx_http_ffi_client.h b/src/ngx_http_ffi_client.h index 5030c09..3255269 100644 --- a/src/ngx_http_ffi_client.h +++ b/src/ngx_http_ffi_client.h @@ -13,6 +13,8 @@ #define NGX_HTTP_FFI_CLIENT_MAX_INTERIM 8 /* fallback size for a pool no caller sized */ #define NGX_HTTP_FFI_CLIENT_KEEPALIVE_POOL_SIZE 30 +/* TLS sessions one worker keeps for resumption, evicted least recently used */ +#define NGX_HTTP_FFI_CLIENT_SSL_SESSION_CACHE_SIZE 256 typedef struct { @@ -47,6 +49,7 @@ typedef struct { ngx_uint_t ssl_verify; ngx_http_ffi_client_ffi_str_t ssl_server_name; ngx_http_ffi_client_ffi_str_t ssl_trusted_certificate; + ngx_uint_t ssl_session_reuse; } ngx_http_ffi_client_ffi_req_t; @@ -77,6 +80,7 @@ typedef struct { ngx_uint_t ssl_verify; ngx_http_ffi_client_ffi_str_t ssl_server_name; ngx_http_ffi_client_ffi_str_t ssl_trusted_certificate; + ngx_uint_t ssl_session_reuse; } ngx_http_ffi_client_ffi_connect_t; @@ -214,6 +218,10 @@ struct ngx_http_ffi_client_ssl_ctx_s { unsigned verify:1; ngx_ssl_t *ssl; }; + +/* the worker's TLS session cache; its layout is private to the session file */ +typedef struct ngx_http_ffi_client_ssl_session_cache_s + ngx_http_ffi_client_ssl_session_cache_t; #endif @@ -222,6 +230,8 @@ typedef struct { ngx_rbtree_node_t sentinel; #if (NGX_SSL) ngx_http_ffi_client_ssl_ctx_t *ssl_ctxs; + ngx_http_ffi_client_ssl_session_cache_t + *ssl_sessions; #endif } ngx_http_ffi_client_main_conf_t; @@ -302,6 +312,7 @@ struct ngx_http_ffi_client_op_s { ngx_str_t ssl_trusted_certificate; unsigned ssl:1; unsigned ssl_verify:1; + unsigned ssl_session_reuse:1; #endif }; @@ -359,6 +370,20 @@ ngx_int_t ngx_http_ffi_client_response_eof(ngx_pool_t *pool, ngx_uint_t ngx_http_ffi_client_has_token(ngx_str_t *value, const char *token, size_t len); +#if (NGX_SSL) +/* + * TLS session resumption. The cache holds one session per TLS peer and lives + * for the worker; enable() arms an SSL_CTX to report new sessions, offer() + * hands the cached one to a fresh connection, and commit() publishes the + * session of a handshake this client accepted. + */ +ngx_int_t ngx_http_ffi_client_ssl_session_cache_enable(ngx_ssl_t *ssl, + ngx_log_t *log); +ngx_int_t ngx_http_ffi_client_ssl_session_offer(ngx_http_ffi_client_op_t *op, + ngx_connection_t *c); +void ngx_http_ffi_client_ssl_session_commit(ngx_connection_t *c); +#endif + ngx_int_t ngx_http_ffi_client_keepalive_bind(ngx_http_request_t *r, ngx_http_ffi_client_op_t *op, ngx_http_ffi_client_ffi_req_t *req, u_char *err, size_t *errlen); diff --git a/src/ngx_http_ffi_client_request.c b/src/ngx_http_ffi_client_request.c index 4bb0292..4b3c3f5 100644 --- a/src/ngx_http_ffi_client_request.c +++ b/src/ngx_http_ffi_client_request.c @@ -135,6 +135,7 @@ ngx_http_ffi_client_start(ngx_http_request_t *r, #if (NGX_SSL) op->ssl = 1; op->ssl_verify = req->ssl_verify ? 1 : 0; + op->ssl_session_reuse = req->ssl_session_reuse ? 1 : 0; if (req->ssl_server_name.len) { if (req->ssl_server_name.data == NULL) { return ngx_http_ffi_client_set_err(err, errlen, @@ -1668,6 +1669,11 @@ ngx_http_ffi_client_ssl_ctx(ngx_http_ffi_client_main_conf_t *fmcf, return NULL; } + if (ngx_http_ffi_client_ssl_session_cache_enable(ssl, log) != NGX_OK) { + ngx_ssl_cleanup_ctx(ssl); + return NULL; + } + /* verify mode stays NONE so the handshake completes and we inspect the * result ourselves. An unusable trust store is a hard init failure: don't * cache a context that would fail every verify-on request. */ @@ -1865,6 +1871,14 @@ ngx_http_ffi_client_ssl_init_connection(ngx_http_ffi_client_op_t *op) return NGX_ERROR; } + if (op->ssl_session_reuse + && ngx_http_ffi_client_ssl_session_offer(op, c) != NGX_OK) + { + ngx_http_ffi_client_finalize(op, NGX_ERROR, + "failed to set up TLS session reuse"); + return NGX_ERROR; + } + op->state = NGX_HTTP_FFI_CLIENT_HANDSHAKING; rc = ngx_ssl_handshake(c); @@ -1925,6 +1939,10 @@ ngx_http_ffi_client_ssl_handshake_handler(ngx_connection_t *c) } } + /* the peer passed every check this caller asked for, so the session it + * established may now be offered to the next connection */ + ngx_http_ffi_client_ssl_session_commit(c); + if (c->read->timer_set) { ngx_del_timer(c->read); } @@ -2286,6 +2304,7 @@ ngx_http_ffi_client_ffi_connect(ngx_http_request_t *r, #if (NGX_SSL) op->ssl = 1; op->ssl_verify = cp->ssl_verify ? 1 : 0; + op->ssl_session_reuse = cp->ssl_session_reuse ? 1 : 0; if (cp->ssl_server_name.len) { if (cp->ssl_server_name.data == NULL) { diff --git a/src/ngx_http_ffi_client_ssl_session.c b/src/ngx_http_ffi_client_ssl_session.c new file mode 100644 index 0000000..158b16a --- /dev/null +++ b/src/ngx_http_ffi_client_ssl_session.c @@ -0,0 +1,515 @@ +#include "ngx_http_ffi_client.h" + + +#if (NGX_SSL) + +/* + * Client-side TLS session resumption. + * + * Every https connection this module opens starts a full handshake unless it + * can offer the server a session from an earlier one. The sessions live in a + * per-worker cache keyed by the TLS peer; a fresh connection offers the entry + * for its key before the handshake and the entry is refilled once the peer + * hands a new session back. + * + * OpenSSL reports client sessions through the new-session callback rather than + * at handshake completion, which is what makes TLS 1.3 work here: its tickets + * arrive after the handshake, on the first read, and a client that only saved + * at handshake time would silently never resume on TLS 1.3. + * + * The connection carries its cache key in SSL ex_data, not a pointer to the + * cache entry: an entry can be evicted and reused for another peer while the + * connection that filled it is still open, and a stale pointer would then + * publish a session under the wrong key. + */ + +typedef struct { + /* keyed by the crc32 of key, so equal hashes are compared in full */ + ngx_rbtree_node_t node; + /* the LRU list while in use, the free list while not */ + ngx_queue_t queue; + ngx_str_t key; + SSL_SESSION *session; +} ngx_http_ffi_client_ssl_session_entry_t; + + +struct ngx_http_ffi_client_ssl_session_cache_s { + ngx_rbtree_t rbtree; + ngx_rbtree_node_t sentinel; + ngx_queue_t lru; + ngx_queue_t free; + ngx_http_ffi_client_ssl_session_entry_t *entries; +}; + + +/* what one connection needs to publish its session, hung off its SSL object */ +typedef struct { + ngx_http_ffi_client_ssl_session_cache_t *cache; + ngx_str_t key; + /* a session the peer sent before the handshake was accepted */ + SSL_SESSION *held; + ngx_log_t *log; + unsigned accepted:1; +} ngx_http_ffi_client_ssl_session_conn_t; + + +static ngx_http_ffi_client_ssl_session_cache_t * + ngx_http_ffi_client_ssl_session_cache(ngx_http_ffi_client_op_t *op); +static ngx_int_t ngx_http_ffi_client_ssl_session_key( + ngx_http_ffi_client_op_t *op, ngx_connection_t *c, ngx_str_t *key); +static ngx_http_ffi_client_ssl_session_entry_t * + ngx_http_ffi_client_ssl_session_lookup( + ngx_http_ffi_client_ssl_session_cache_t *cache, ngx_str_t *key); +static ngx_http_ffi_client_ssl_session_entry_t * + ngx_http_ffi_client_ssl_session_lookup_node(ngx_rbtree_node_t *node, + ngx_rbtree_node_t *sentinel, ngx_rbtree_key_t hash, ngx_str_t *key); +static void ngx_http_ffi_client_ssl_session_store( + ngx_http_ffi_client_ssl_session_conn_t *sc, SSL_SESSION *session); +static void ngx_http_ffi_client_ssl_session_drop( + ngx_http_ffi_client_ssl_session_cache_t *cache, + ngx_http_ffi_client_ssl_session_entry_t *entry); +static ngx_uint_t ngx_http_ffi_client_ssl_session_expired( + SSL_SESSION *session); +static int ngx_http_ffi_client_ssl_session_new(ngx_ssl_conn_t *ssl_conn, + SSL_SESSION *session); +static void ngx_http_ffi_client_ssl_session_cleanup(void *data); + + +/* SSL ex_data slot holding ngx_http_ffi_client_ssl_session_conn_t */ +static int ngx_http_ffi_client_ssl_session_index = -1; + + +/* + * Arm a client SSL_CTX to report the sessions its connections are given. + * NO_INTERNAL keeps OpenSSL from holding its own copies: this cache is the only + * one, so eviction and expiry stay in one place. + */ +ngx_int_t +ngx_http_ffi_client_ssl_session_cache_enable(ngx_ssl_t *ssl, ngx_log_t *log) +{ + if (ngx_http_ffi_client_ssl_session_index == -1) { + ngx_http_ffi_client_ssl_session_index = + SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL); + + if (ngx_http_ffi_client_ssl_session_index == -1) { + ngx_ssl_error(NGX_LOG_ALERT, log, 0, + "SSL_get_ex_new_index() failed"); + return NGX_ERROR; + } + } + + SSL_CTX_set_session_cache_mode(ssl->ctx, + SSL_SESS_CACHE_CLIENT + |SSL_SESS_CACHE_NO_INTERNAL); + + SSL_CTX_sess_set_new_cb(ssl->ctx, ngx_http_ffi_client_ssl_session_new); + + return NGX_OK; +} + + +/* + * Offer this connection the cached session for its peer, and leave behind what + * the new-session callback needs to fill the entry again. + */ +ngx_int_t +ngx_http_ffi_client_ssl_session_offer(ngx_http_ffi_client_op_t *op, + ngx_connection_t *c) +{ + ngx_str_t key; + ngx_pool_cleanup_t *cln; + ngx_http_ffi_client_ssl_session_conn_t *sc; + ngx_http_ffi_client_ssl_session_cache_t *cache; + ngx_http_ffi_client_ssl_session_entry_t *entry; + + cache = ngx_http_ffi_client_ssl_session_cache(op); + if (cache == NULL) { + return NGX_ERROR; + } + + if (ngx_http_ffi_client_ssl_session_key(op, c, &key) != NGX_OK) { + return NGX_ERROR; + } + + sc = ngx_pcalloc(c->pool, sizeof(ngx_http_ffi_client_ssl_session_conn_t)); + if (sc == NULL) { + return NGX_ERROR; + } + + sc->cache = cache; + sc->key = key; + sc->log = ngx_cycle->log; + + /* a connection that dies before its session is published still owns it */ + cln = ngx_pool_cleanup_add(c->pool, 0); + if (cln == NULL) { + return NGX_ERROR; + } + + cln->handler = ngx_http_ffi_client_ssl_session_cleanup; + cln->data = sc; + + if (SSL_set_ex_data(c->ssl->connection, + ngx_http_ffi_client_ssl_session_index, sc) + == 0) + { + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "SSL_set_ex_data() failed"); + return NGX_ERROR; + } + + entry = ngx_http_ffi_client_ssl_session_lookup(cache, &key); + if (entry == NULL) { + return NGX_OK; + } + + if (ngx_http_ffi_client_ssl_session_expired(entry->session)) { + ngx_http_ffi_client_ssl_session_drop(cache, entry); + return NGX_OK; + } + + /* a session the peer no longer knows only costs a full handshake, so a + * refused one is not worth failing the request over */ + (void) ngx_ssl_set_session(c, entry->session); + + ngx_queue_remove(&entry->queue); + ngx_queue_insert_head(&cache->lru, &entry->queue); + + return NGX_OK; +} + + +/* + * The handshake was accepted: publish its session. Until this runs a session is + * only held, so a peer this client rejected never leaves one behind for the + * next request to resume. + */ +void +ngx_http_ffi_client_ssl_session_commit(ngx_connection_t *c) +{ + SSL_SESSION *session; + ngx_http_ffi_client_ssl_session_conn_t *sc; + + if (ngx_http_ffi_client_ssl_session_index == -1 + || c->ssl == NULL + || c->ssl->connection == NULL) + { + return; + } + + sc = SSL_get_ex_data(c->ssl->connection, + ngx_http_ffi_client_ssl_session_index); + if (sc == NULL) { + return; + } + + sc->accepted = 1; + + session = sc->held; + if (session == NULL) { + return; + } + + sc->held = NULL; + ngx_http_ffi_client_ssl_session_store(sc, session); +} + + +/* The worker's cache, built on first use and kept for the life of the worker. */ +static ngx_http_ffi_client_ssl_session_cache_t * +ngx_http_ffi_client_ssl_session_cache(ngx_http_ffi_client_op_t *op) +{ + ngx_uint_t i; + ngx_http_ffi_client_main_conf_t *fmcf; + ngx_http_ffi_client_ssl_session_cache_t *cache; + + fmcf = ngx_http_get_module_main_conf(op->r, ngx_http_ffi_client_module); + if (fmcf == NULL) { + return NULL; + } + + if (fmcf->ssl_sessions != NULL) { + return fmcf->ssl_sessions; + } + + cache = ngx_pcalloc(ngx_cycle->pool, + sizeof(ngx_http_ffi_client_ssl_session_cache_t)); + if (cache == NULL) { + return NULL; + } + + cache->entries = ngx_pcalloc(ngx_cycle->pool, + sizeof(ngx_http_ffi_client_ssl_session_entry_t) + * NGX_HTTP_FFI_CLIENT_SSL_SESSION_CACHE_SIZE); + if (cache->entries == NULL) { + return NULL; + } + + ngx_rbtree_init(&cache->rbtree, &cache->sentinel, ngx_rbtree_insert_value); + ngx_queue_init(&cache->lru); + ngx_queue_init(&cache->free); + + for (i = 0; i < NGX_HTTP_FFI_CLIENT_SSL_SESSION_CACHE_SIZE; i++) { + ngx_queue_insert_head(&cache->free, &cache->entries[i].queue); + } + + fmcf->ssl_sessions = cache; + + return cache; +} + + +/* + * The cache key: verify mode, peer address, SNI, trusted CA. + * + * A resumed handshake skips the certificate exchange and answers + * SSL_get_verify_result() from the session, so every input the handshake was + * judged on has to be part of the key. That makes it at least as strict as the + * keepalive pool key, which is tagged the same way except that it groups by the + * caller's pool name; a session is keyed by the address actually connected to, + * so a pool that spans peers still resumes each one only against itself. + * + * The CA is folded away with verify off, matching the SSL_CTX cache: nothing + * reads it there either, and keying on it would split the cache per caller for + * no gain. + */ +static ngx_int_t +ngx_http_ffi_client_ssl_session_key(ngx_http_ffi_client_op_t *op, + ngx_connection_t *c, ngx_str_t *key) +{ + u_char *p; + size_t len, addr_len, name_len, ca_len; + + addr_len = op->peer_name.len; + name_len = op->ssl_server_name.len; + ca_len = op->ssl_verify ? op->ssl_trusted_certificate.len : 0; + + /* tagged, length-prefixed so no two different keys can encode alike */ + len = 1 + sizeof(size_t) + addr_len + sizeof(size_t) + name_len + + sizeof(size_t) + ca_len; + + key->data = ngx_pnalloc(c->pool, len); + if (key->data == NULL) { + return NGX_ERROR; + } + + p = key->data; + *p++ = op->ssl_verify ? 1 : 0; + + p = ngx_cpymem(p, &addr_len, sizeof(size_t)); + if (addr_len) { + p = ngx_cpymem(p, op->peer_name.data, addr_len); + } + + p = ngx_cpymem(p, &name_len, sizeof(size_t)); + if (name_len) { + p = ngx_cpymem(p, op->ssl_server_name.data, name_len); + } + + p = ngx_cpymem(p, &ca_len, sizeof(size_t)); + if (ca_len) { + p = ngx_cpymem(p, op->ssl_trusted_certificate.data, ca_len); + } + + key->len = p - key->data; + + return NGX_OK; +} + + +static ngx_http_ffi_client_ssl_session_entry_t * +ngx_http_ffi_client_ssl_session_lookup( + ngx_http_ffi_client_ssl_session_cache_t *cache, ngx_str_t *key) +{ + ngx_rbtree_key_t hash; + + hash = ngx_crc32_short(key->data, key->len); + + return ngx_http_ffi_client_ssl_session_lookup_node(cache->rbtree.root, + cache->rbtree.sentinel, + hash, key); +} + + +static ngx_http_ffi_client_ssl_session_entry_t * +ngx_http_ffi_client_ssl_session_lookup_node(ngx_rbtree_node_t *node, + ngx_rbtree_node_t *sentinel, ngx_rbtree_key_t hash, ngx_str_t *key) +{ + ngx_http_ffi_client_ssl_session_entry_t *entry; + + if (node == sentinel) { + return NULL; + } + + if (hash < node->key) { + return ngx_http_ffi_client_ssl_session_lookup_node(node->left, sentinel, + hash, key); + } + + if (hash > node->key) { + return ngx_http_ffi_client_ssl_session_lookup_node(node->right, sentinel, + hash, key); + } + + entry = (ngx_http_ffi_client_ssl_session_entry_t *) node; + + if (entry->key.len == key->len + && ngx_memcmp(entry->key.data, key->data, key->len) == 0) + { + return entry; + } + + /* equal hashes, different keys: both subtrees may hold the match */ + entry = ngx_http_ffi_client_ssl_session_lookup_node(node->left, sentinel, + hash, key); + if (entry != NULL) { + return entry; + } + + return ngx_http_ffi_client_ssl_session_lookup_node(node->right, sentinel, + hash, key); +} + + +/* + * Put a session in the cache under this connection's key, taking over the + * reference the caller holds. A full cache gives up its least recently used + * entry. + */ +static void +ngx_http_ffi_client_ssl_session_store( + ngx_http_ffi_client_ssl_session_conn_t *sc, SSL_SESSION *session) +{ + ngx_queue_t *q; + ngx_http_ffi_client_ssl_session_cache_t *cache; + ngx_http_ffi_client_ssl_session_entry_t *entry; + + cache = sc->cache; + + entry = ngx_http_ffi_client_ssl_session_lookup(cache, &sc->key); + + if (entry != NULL) { + SSL_SESSION_free(entry->session); + entry->session = session; + + ngx_queue_remove(&entry->queue); + ngx_queue_insert_head(&cache->lru, &entry->queue); + + return; + } + + if (ngx_queue_empty(&cache->free)) { + q = ngx_queue_last(&cache->lru); + entry = ngx_queue_data(q, ngx_http_ffi_client_ssl_session_entry_t, + queue); + ngx_http_ffi_client_ssl_session_drop(cache, entry); + } + + q = ngx_queue_head(&cache->free); + entry = ngx_queue_data(q, ngx_http_ffi_client_ssl_session_entry_t, queue); + + /* the key outlives the connection it was built on, so it gets its own + * allocation, freed when the entry is dropped */ + entry->key.data = ngx_alloc(sc->key.len, sc->log); + if (entry->key.data == NULL) { + SSL_SESSION_free(session); + return; + } + + ngx_memcpy(entry->key.data, sc->key.data, sc->key.len); + entry->key.len = sc->key.len; + entry->session = session; + entry->node.key = ngx_crc32_short(entry->key.data, entry->key.len); + + ngx_rbtree_insert(&cache->rbtree, &entry->node); + + ngx_queue_remove(q); + ngx_queue_insert_head(&cache->lru, q); +} + + +/* Empty an entry and hand it back to the free list. */ +static void +ngx_http_ffi_client_ssl_session_drop( + ngx_http_ffi_client_ssl_session_cache_t *cache, + ngx_http_ffi_client_ssl_session_entry_t *entry) +{ + SSL_SESSION_free(entry->session); + entry->session = NULL; + + ngx_free(entry->key.data); + entry->key.data = NULL; + entry->key.len = 0; + + ngx_rbtree_delete(&cache->rbtree, &entry->node); + + ngx_queue_remove(&entry->queue); + ngx_queue_insert_head(&cache->free, &entry->queue); +} + + +/* + * Offering a session the peer has timed out only costs a round trip, but an + * entry nothing asks for again would sit there for the life of the worker, so + * expiry is checked where it is cheap: on the way out. + */ +static ngx_uint_t +ngx_http_ffi_client_ssl_session_expired(SSL_SESSION *session) +{ + time_t started; + +#if (OPENSSL_VERSION_NUMBER >= 0x30400000L && !defined LIBRESSL_VERSION_NUMBER) + started = SSL_SESSION_get_time_ex(session); +#else + started = (time_t) SSL_SESSION_get_time(session); +#endif + + return started + (time_t) SSL_SESSION_get_timeout(session) <= ngx_time(); +} + + +/* + * OpenSSL hands over a session: at handshake completion on TLS 1.2, and with + * every ticket the peer sends afterwards on TLS 1.3. Returning 1 keeps the + * reference, which the cache then owns. + */ +static int +ngx_http_ffi_client_ssl_session_new(ngx_ssl_conn_t *ssl_conn, + SSL_SESSION *session) +{ + ngx_http_ffi_client_ssl_session_conn_t *sc; + + sc = SSL_get_ex_data(ssl_conn, ngx_http_ffi_client_ssl_session_index); + + /* no ex_data: this caller turned resumption off */ + if (sc == NULL) { + return 0; + } + +#if (OPENSSL_VERSION_NUMBER >= 0x10101000L && !defined LIBRESSL_VERSION_NUMBER) + if (!SSL_SESSION_is_resumable(session)) { + return 0; + } +#endif + + if (!sc->accepted) { + SSL_SESSION_free(sc->held); + sc->held = session; + return 1; + } + + ngx_http_ffi_client_ssl_session_store(sc, session); + + return 1; +} + + +static void +ngx_http_ffi_client_ssl_session_cleanup(void *data) +{ + ngx_http_ffi_client_ssl_session_conn_t *sc = data; + + SSL_SESSION_free(sc->held); + sc->held = NULL; +} + +#endif diff --git a/t/016-tls-session-resumption.t b/t/016-tls-session-resumption.t new file mode 100644 index 0000000..8780a61 --- /dev/null +++ b/t/016-tls-session-resumption.t @@ -0,0 +1,409 @@ +use Test::Nginx::Socket -Base; + +repeat_each(1); +plan tests => repeat_each() * (blocks() * 3); + +our $HttpConfig = qq{ + lua_package_path "$ENV{TEST_NGINX_LUA_PACKAGE_PATH};;"; + + # TLS 1.3 hands its session out after the handshake, in a ticket + server { + listen 127.0.0.1:12799 ssl; + ssl_certificate \$TEST_NGINX_SERVER_ROOT/html/tls.crt; + ssl_certificate_key \$TEST_NGINX_SERVER_ROOT/html/tls.key; + ssl_protocols TLSv1.3; + ssl_session_cache shared:ffi13:1m; + ssl_session_timeout 10m; + + location / { + content_by_lua_block { + local body = "secure\\n" + ngx.header["Content-Length"] = #body + ngx.header["X-Reused"] = ngx.var.ssl_session_reused + ngx.print(body) + } + } + } + + # TLS 1.2 hands it out during the handshake + server { + listen 127.0.0.1:12798 ssl; + ssl_certificate \$TEST_NGINX_SERVER_ROOT/html/tls.crt; + ssl_certificate_key \$TEST_NGINX_SERVER_ROOT/html/tls.key; + ssl_protocols TLSv1.2; + ssl_session_cache shared:ffi12:1m; + ssl_session_timeout 10m; + + location / { + content_by_lua_block { + local body = "secure\\n" + ngx.header["Content-Length"] = #body + ngx.header["X-Reused"] = ngx.var.ssl_session_reused + ngx.print(body) + } + } + } +}; + +our $UserFiles = <<'_EOC_'; +>>> tls.crt +-----BEGIN CERTIFICATE----- +MIIDKjCCAhKgAwIBAgIUMpoLE+76X2UdJn+TC1k80Qr6O0MwDQYJKoZIhvcNAQEL +BQAwFTETMBEGA1UEAwwKdGVzdC5sb2NhbDAgFw0yNjA3MTUwODU2MDRaGA8yMTI2 +MDYyMTA4NTYwNFowFTETMBEGA1UEAwwKdGVzdC5sb2NhbDCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAMWo0WdpxTqswzXDnhsRpqby32slAR4BmzjXECvo +LHK1Oza+/8SCAo9r61HXl5fyy6cCll75MzEmZkLJy8vwmvCh5qTBBDt68eq6RbGH +FRlCNhjup2KvsXoUWfmOQC72PMYKg/faY8RoV40P9x/W57JvT5WSCf6O3/anre8g +ywokpH3QokPRdIEg3OE+OTAMcDV43mISXjWJWAtpJU71T/i0Iyxy5IBU01qXAjEe +WIy8fQLzPoK4WqiHH6jII9bB6ebDrWl5SW2CmrStb5lnMuf8OqXXomXBzTcfC3cu +CAREN2a3uZiAPryG4paTBD/3+dmyLDmjEmeFV2CQCSJ9bQkCAwEAAaNwMG4wHQYD +VR0OBBYEFIZZdTC230/zMtPUjW1CpizXxWGGMB8GA1UdIwQYMBaAFIZZdTC230/z +MtPUjW1CpizXxWGGMA8GA1UdEwEB/wQFMAMBAf8wGwYDVR0RBBQwEoIKdGVzdC5s +b2NhbIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAYv+q+iaGoJG1qStGFk/Ojhpa +aLXti7QV53aCaUpNbiKiCxtIC1ZqpaQAW+G5eM2dWC+7TCrqcMJuJjjp5c9mwolR +WHMqinfNUe/LCa3DsJvqYEkr1Dmy+iPabfP6Ni3UnWMehehTvX86jd4GjR7mTj6/ +PC2D6x5gy+SFQSh+OMvtMkmYM91s21UIds5z9PhXDVzolH3kP3zaqtu646F1btIK +g5sdzjzunwEd5Av6hzaqYNu/ysbz6NiVcTOj8cA3RaktwvgBmWE2mRim2BO5bAXf +JE9Vloa+pp9bLzcwlTaU2zIZb2SadrMbNaNdh8lJ/iX5H3pSaNODNTThmw6BWA== +-----END CERTIFICATE----- +>>> tls.key +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDFqNFnacU6rMM1 +w54bEaam8t9rJQEeAZs41xAr6CxytTs2vv/EggKPa+tR15eX8sunApZe+TMxJmZC +ycvL8JrwoeakwQQ7evHqukWxhxUZQjYY7qdir7F6FFn5jkAu9jzGCoP32mPEaFeN +D/cf1ueyb0+Vkgn+jt/2p63vIMsKJKR90KJD0XSBINzhPjkwDHA1eN5iEl41iVgL +aSVO9U/4tCMscuSAVNNalwIxHliMvH0C8z6CuFqohx+oyCPWwenmw61peUltgpq0 +rW+ZZzLn/Dql16Jlwc03Hwt3LggERDdmt7mYgD68huKWkwQ/9/nZsiw5oxJnhVdg +kAkifW0JAgMBAAECggEAIAgKG1ygKjCKGAHh8tgK7j4op6fhBPhUq8Lqa3seDN7C +wE32i+VXvd9KzMIH3odpqmB4dt6ihaIH62XhYWTV7w4Fnwhqg6saXiQenDTcXfIF +a0ftl0gKllKK/C6pxxJ/acaVeUqKZW9VVNZUAXRlqtxwBLicZwTHVaT5wmlJjhR2 +LA0+zqup5J4SxDAk9+p1SgykxhEPP/JkAyAtD39jeQ8uJWx/Rnq3SDLseGUs8qUM +sgHsa5PSHxEmnuM7/LOkO3v6EbkAQsy8hcRdBrXt5DUr1LSk7cmXQlODkQ1klpIU +lAbd+snLarh4+bHzbCfY2PoYy/uqYx7xmBc+FHsE8wKBgQDjj0d3b7KYX2jJWkuT +5o2HD9koRSTrhvuM9/fsYCfJ24QDkUF45OlSd61z7Q9BcfSp8KLuIVbnFOSP4Gim +ZZqOQu0wZffaBWVeXwBpCWNXO/8xuufPXIGaezH4IXZN/uIZWGY5PHHPgACLYSnf +6RXoTTMRCmyWwLip/8l1fCZYQwKBgQDeXOMtHttRaqXCgZY2vkJFd2MqYx+TzREK +1Nzna3vSrsMqvWrwxaONQQK6hphtrfxaB8DPhMWl+0bsrgrjB/7aLDDj7BxUeZPt +/trXRKE+rVMfXKAewvM1NO9to91pDjOOamkG7ih8gWfR4jE6qE0ONvCFwoDS6hQq +XsRUuFTmwwKBgQC4y7Eoz/+D9+8bnQVFLXR/WyJproUF888yMmkWfxuwtGBnmT1H +FPZZbzDftIKwDf+3ReC6az6sV+4o3P9/KYGyx6zgod3+ImWolpO5uNMAk4tw8iyv +25qwPh1dOKdfPX6VQJF7J5fw/yzyA0zDNgEBbjfrPcDjR8xu2Xbbvp9RCwKBgFG+ ++jFTP7N9rnSEKVH0ve5FxqoFiM1QPSyrNo7JH9tDLjKfMhpTvh2mwbcK1iy0IqqC +YSqpF/Q+HUPTc+MkxFc2mb6gxYV0sKJ058Tt0Q12sLE93wuQBdMQo9i9vh7p/qAj +lHrcwPuMozswmYKD7tgD8IZsC+n97e3pqumuXl/7AoGAV88zb7UorfzYmilI12dR +dMEmcapTrGz0WTFrAnkMPOMo4haBswySqePfrG5oC/xa77Dw6/LOD8b1pmupeKvx +YrErqu4suZWhAa1m2pblIxAM3CA3l5ZZR8rF4w0Iau1OoUHDEATOTYETiitKuvFp +Ye/gbwb0N+OIZRCBROm7A70= +-----END PRIVATE KEY----- +_EOC_ + + +no_long_string(); +run_tests(); + +__DATA__ + +=== TEST 1: TLS 1.3 resumes the session of an earlier connection +--- http_config eval: $::HttpConfig +--- user_files eval: $::UserFiles +--- config + location /t { + content_by_lua_block { + local client = require "resty.ngx_http_ffi_client" + + for _ = 1, 2 do + local res, err = client.request_uri({ + scheme = "https", + host = "127.0.0.1", + port = 12799, + method = "GET", + path = "/", + ssl_verify = false, + }) + + if not res then + ngx.status = 500 + ngx.say(err) + return + end + + ngx.say(res.status, " ", res.headers["X-Reused"]) + end + } + } +--- request +GET /t +--- response_body +200 . +200 r +--- no_error_log +[error] + + + +=== TEST 2: TLS 1.2 resumes the session of an earlier connection +--- http_config eval: $::HttpConfig +--- user_files eval: $::UserFiles +--- config + location /t { + content_by_lua_block { + local client = require "resty.ngx_http_ffi_client" + + for _ = 1, 2 do + local res, err = client.request_uri({ + scheme = "https", + host = "127.0.0.1", + port = 12798, + method = "GET", + path = "/", + ssl_verify = false, + }) + + if not res then + ngx.status = 500 + ngx.say(err) + return + end + + ngx.say(res.status, " ", res.headers["X-Reused"]) + end + } + } +--- request +GET /t +--- response_body +200 . +200 r +--- no_error_log +[error] + + + +=== TEST 3: ssl_session_reuse = false keeps every handshake full +--- http_config eval: $::HttpConfig +--- user_files eval: $::UserFiles +--- config + location /t { + content_by_lua_block { + local client = require "resty.ngx_http_ffi_client" + + for _ = 1, 3 do + local res, err = client.request_uri({ + scheme = "https", + host = "127.0.0.1", + port = 12799, + method = "GET", + path = "/", + ssl_verify = false, + ssl_session_reuse = false, + }) + + if not res then + ngx.status = 500 + ngx.say(err) + return + end + + ngx.say(res.status, " ", res.headers["X-Reused"]) + end + } + } +--- request +GET /t +--- response_body +200 . +200 . +200 . +--- no_error_log +[error] + + + +=== TEST 4: a verify-off session is never offered to a verify-on request +--- http_config eval: $::HttpConfig +--- user_files eval: $::UserFiles +--- config + location /t { + content_by_lua_block { + local client = require "resty.ngx_http_ffi_client" + + -- same peer and same SNI throughout: the verify flag is the only + -- thing telling the two sessions apart + local function get(verify) + local res, err = client.request_uri({ + scheme = "https", + host = "127.0.0.1", + port = 12799, + method = "GET", + path = "/", + ssl_verify = verify, + ssl_server_name = "test.local", + ssl_trusted_certificate = + "$TEST_NGINX_SERVER_ROOT/html/tls.crt", + }) + + if not res then + ngx.status = 500 + ngx.say(err) + return false + end + + ngx.say(res.status, " ", res.headers["X-Reused"]) + return true + end + + if not get(false) then return end + if not get(false) then return end + if not get(true) then return end + get(true) + } + } +--- request +GET /t +--- response_body +200 . +200 r +200 . +200 r +--- no_error_log +[error] + + + +=== TEST 5: the stateful object resumes across connections +--- http_config eval: $::HttpConfig +--- user_files eval: $::UserFiles +--- config + location /t { + content_by_lua_block { + local client = require "resty.ngx_http_ffi_client" + + for _ = 1, 2 do + local httpc = client.new() + httpc:set_timeouts(2000, 2000, 2000) + + local ok, err = httpc:connect({ + scheme = "https", + host = "127.0.0.1", + port = 12799, + ssl_verify = false, + }) + + if not ok then + ngx.status = 500 + ngx.say(err) + return + end + + local res + res, err = httpc:request({ method = "GET", path = "/", + preread_body = true }) + if not res then + ngx.status = 500 + ngx.say(err) + return + end + + ngx.say(res.status, " ", res.headers["X-Reused"]) + httpc:close() + end + } + } +--- request +GET /t +--- response_body +200 . +200 r +--- no_error_log +[error] + + +=== TEST 6: sessions do not cross SNI +--- http_config eval: $::HttpConfig +--- user_files eval: $::UserFiles +--- config + location /t { + content_by_lua_block { + local client = require "resty.ngx_http_ffi_client" + + local function get(name) + local res, err = client.request_uri({ + scheme = "https", + host = "127.0.0.1", + port = 12799, + method = "GET", + path = "/", + ssl_verify = false, + ssl_server_name = name, + }) + + if not res then + ngx.status = 500 + ngx.say(err) + return false + end + + ngx.say(name, " ", res.headers["X-Reused"]) + return true + end + + if not get("one.test") then return end + if not get("two.test") then return end + get("one.test") + } + } +--- request +GET /t +--- response_body +one.test . +two.test . +one.test r +--- no_error_log +[error] + + + +=== TEST 7: the cache is bounded and drops its least recently used peer +--- http_config eval: $::HttpConfig +--- user_files eval: $::UserFiles +--- config + location /t { + content_by_lua_block { + local client = require "resty.ngx_http_ffi_client" + + -- one entry per SNI, so 260 peers overrun a cache that holds 256 + local function get(name) + local res, err = client.request_uri({ + scheme = "https", + host = "127.0.0.1", + port = 12799, + method = "GET", + path = "/", + ssl_verify = false, + ssl_server_name = name, + }) + + if not res then + ngx.status = 500 + ngx.say(err) + return nil + end + + return res.headers["X-Reused"] + end + + for i = 1, 260 do + if not get("h" .. i .. ".test") then return end + end + + ngx.say("first ", get("h1.test")) + ngx.say("last ", get("h260.test")) + } + } +--- request +GET /t +--- response_body +first . +last r +--- no_error_log +[error] From d1c4429c29b2da6a66ffd0f7747872fcc00a28af Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Thu, 6 Aug 2026 07:54:01 +0545 Subject: [PATCH 2/2] fix: degrade instead of failing when session reuse cannot be set up Nothing ngx_http_ffi_client_ssl_session_offer() can fail on stops a normal handshake, and the full handshake it falls back to is what the connection would have done anyway, so a setup failure is logged rather than failing the request. The cache test now fills to exactly the cap, uses its oldest peer, and adds one more: the peer that goes has to be the second oldest, which a cache that never promotes on a hit would keep while dropping the one just used. --- README.md | 11 ++++++----- src/ngx_http_ffi_client_request.c | 7 ++++--- t/016-tls-session-resumption.t | 23 +++++++++++++++++------ 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8e36afa..b19ca76 100644 --- a/README.md +++ b/README.md @@ -106,11 +106,12 @@ same connections from either entry point. ### TLS session resumption An https connection offers the server the session from an earlier handshake with -the same peer, so it costs one round trip instead of two and no certificate work. -Both entry points do it, and both take `ssl_session_reuse = false` to turn it -off, the way nginx's `proxy_ssl_session_reuse` does. It matters most where the -connection pool cannot help: a fresh connection per request, which is what the -`tlsshort` benchmark shape measures. +the same peer. A resumed handshake sends no certificate and does no signature or +key-exchange work, and on TLS 1.2 it also saves a round trip. Both entry points +do it, and both take `ssl_session_reuse = false` to turn it off, the way nginx's +`proxy_ssl_session_reuse` does. It matters most where the connection pool cannot +help: a fresh connection per request, which is what the `tlsshort` benchmark +shape measures. - The cache holds one session per TLS peer and lives for the worker, up to `256` peers, evicting the least recently used. A session past its expiry is dropped diff --git a/src/ngx_http_ffi_client_request.c b/src/ngx_http_ffi_client_request.c index 4b3c3f5..4000662 100644 --- a/src/ngx_http_ffi_client_request.c +++ b/src/ngx_http_ffi_client_request.c @@ -1871,12 +1871,13 @@ ngx_http_ffi_client_ssl_init_connection(ngx_http_ffi_client_op_t *op) return NGX_ERROR; } + /* resumption is an optimization, and what it fails back to is the full + * handshake this connection would have done anyway */ if (op->ssl_session_reuse && ngx_http_ffi_client_ssl_session_offer(op, c) != NGX_OK) { - ngx_http_ffi_client_finalize(op, NGX_ERROR, - "failed to set up TLS session reuse"); - return NGX_ERROR; + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "ffi client TLS session reuse unavailable"); } op->state = NGX_HTTP_FFI_CLIENT_HANDSHAKING; diff --git a/t/016-tls-session-resumption.t b/t/016-tls-session-resumption.t index 8780a61..d3c4beb 100644 --- a/t/016-tls-session-resumption.t +++ b/t/016-tls-session-resumption.t @@ -371,7 +371,7 @@ one.test r content_by_lua_block { local client = require "resty.ngx_http_ffi_client" - -- one entry per SNI, so 260 peers overrun a cache that holds 256 + -- one entry per SNI, so the peer count is the cache pressure local function get(name) local res, err = client.request_uri({ scheme = "https", @@ -392,18 +392,29 @@ one.test r return res.headers["X-Reused"] end - for i = 1, 260 do + -- fill the cache exactly: h1 is now its least recently used peer + for i = 1, 256 do if not get("h" .. i .. ".test") then return end end - ngx.say("first ", get("h1.test")) - ngx.say("last ", get("h260.test")) + -- h1 still resumes, and using it makes h2 the least recently used + ngx.say("h1 ", get("h1.test")) + + -- one peer past the cap, so exactly one entry has to go + ngx.say("h257 ", get("h257.test")) + + -- the one that went is h2, not the h1 an unpromoted cache would + -- have dropped + ngx.say("h1 ", get("h1.test")) + ngx.say("h2 ", get("h2.test")) } } --- request GET /t --- response_body -first . -last r +h1 r +h257 . +h1 r +h2 . --- no_error_log [error]