Rebase ClickHouse patches onto openssl-3.5.8

.gitmodules

@@ -1,37 +0,0 @@
-[submodule "pyca.cryptography"]
-	path = pyca-cryptography
-	url = https://github.com/pyca/cryptography.git
-
-[submodule "krb5"]
-	path = krb5
-	url = https://github.com/krb5/krb5
-
-[submodule "gost-engine"]
-	path = gost-engine
-	url = https://github.com/gost-engine/engine
-	update = rebase
-[submodule "wycheproof"]
-	path = wycheproof
-	url = https://github.com/google/wycheproof
-[submodule "tlsfuzzer"]
-	path = tlsfuzzer
-	url = https://github.com/tlsfuzzer/tlsfuzzer
-[submodule "python-ecdsa"]
-	path = python-ecdsa
-	url = https://github.com/tlsfuzzer/python-ecdsa
-[submodule "tlslite-ng"]
-	path = tlslite-ng
-	url = https://github.com/tlsfuzzer/tlslite-ng
-[submodule "oqs-provider"]
-	path = oqs-provider
-	url = https://github.com/open-quantum-safe/oqs-provider.git
-[submodule "cloudflare-quiche"]
-	path = cloudflare-quiche
-	url = https://github.com/cloudflare/quiche
-[submodule "fuzz/corpora"]
-	path = fuzz/corpora
-	url = https://github.com/openssl/fuzz-corpora
-	branch = main
-[submodule "pkcs11-provider"]
-	path = pkcs11-provider
-	url = https://github.com/latchset/pkcs11-provider.git

crypto/armcap.c

@@ -77,6 +77,10 @@ void OPENSSL_cpuid_setup(void) __attribute__((constructor));
 #include <sys/auxv.h>
 #define OSSL_IMPLEMENT_GETAUXVAL
 #endif
+#elif defined(__MUSL__)
+/* musl has always provided getauxval(), unversioned. */
+#include <sys/auxv.h>
+#define OSSL_IMPLEMENT_GETAUXVAL
 #endif
 #if defined(__FreeBSD__) || defined(__OpenBSD__)
 #include <sys/param.h>

crypto/bio/bss_dgram.c

@@ -49,15 +49,17 @@
 #define M_METHOD_WSARECVMSG 4
 
 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
-#if !(__GLIBC_PREREQ(2, 14))
+/// ClickHouse-specific patch: Pretend to use a stone age glibc because we use a stone age glibc.
+/// Otherwise, system calls sendmmsg and recvmmsg are used which work only with too-new glibc 2.14.
+/// #if !(__GLIBC_PREREQ(2, 14))
 #undef NO_RECVMMSG
 /*
  * Some old glibc versions may have recvmmsg and MSG_WAITFORONE flag, but
  * not sendmmsg. We need both so force this to be disabled on these old
  * versions
  */
 #define NO_RECVMMSG
-#endif
+/// #endif
 #endif
 #if defined(__GNU__)
 /* GNU/Hurd does not have IP_PKTINFO yet */

crypto/bn/bn_intern.c

@@ -10,6 +10,11 @@
 #include "internal/cryptlib.h"
 #include "bn_local.h"
 
+#if defined(__has_feature)
+# if __has_feature(memory_sanitizer)
+#  include <sanitizer/msan_interface.h>
+# endif
+#endif
 /*
  * Determine the modified width-(w+1) Non-Adjacent Form (wNAF) of 'scalar'.
  * This is an array  r[]  of values that are either zero or odd with an
@@ -188,6 +193,11 @@ int bn_set_words(BIGNUM *a, const BN_ULONG *words, int num_words)
         return 0;
     }
 
+#if defined(__has_feature)
+# if __has_feature(memory_sanitizer)
+    __msan_unpoison(words, sizeof(BN_ULONG) * num_words);
+# endif
+#endif
     memcpy(a->d, words, sizeof(BN_ULONG) * num_words);
     a->top = num_words;
     bn_correct_top(a);

crypto/cpuid.c

@@ -102,7 +102,16 @@ void OPENSSL_cpuid_setup(void)
     if (trigger)
         return;
 
+    /// This function is called from .init section before memory sanitizer mmaps shadow memory.
+    /// Program will crash with segmentation fault when trying access `trigger`,
+    /// because its address was replaced with some not mapped address.
+    /// Also see https://github.com/ClickHouse/openssl/pull/5
+    /// Unfortunately, __msan_init() is no longer part of msan's public header and there seems to be no replacement.
+#if defined(__has_feature)
+# if !__has_feature(memory_sanitizer)
     trigger = 1;
+# endif
+#endif
     if ((env = ossl_getenv("OPENSSL_ia32cap")) != NULL) {
         int off = (env[0] == '~') ? 1 : 0;
 

crypto/ec/curve25519.c

@@ -21,6 +21,12 @@
 
 #include "internal/numbers.h"
 
+#if defined(__has_feature)
+# if __has_feature(memory_sanitizer)
+#  include <sanitizer/msan_interface.h>
+# endif
+#endif
+
 #if defined(X25519_ASM) && (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64))
 
 #define BASE_2_64_IMPLEMENTED
@@ -5846,6 +5852,12 @@ int ossl_x25519(uint8_t out_shared_key[32], const uint8_t private_key[32],
 {
     static const uint8_t kZeros[32] = { 0 };
     x25519_scalar_mult(out_shared_key, private_key, peer_public_value);
+#if defined(__has_feature)
+# if __has_feature(memory_sanitizer)
+    /* x25519_scalar_mult may use assembly that MSan cannot instrument. */
+    __msan_unpoison(out_shared_key, 32);
+# endif
+#endif
     /* The all-zero output results when the input is a point of small order. */
     return CRYPTO_memcmp(kZeros, out_shared_key, 32) != 0;
 }
@@ -5875,5 +5887,11 @@ void ossl_x25519_public_from_private(uint8_t out_public_value[32],
     fe_mul(zplusy, zplusy, zminusy_inv);
     fe_tobytes(out_public_value, zplusy);
 
+#if defined(__has_feature)
+# if __has_feature(memory_sanitizer)
+    __msan_unpoison(out_public_value, 32);
+# endif
+#endif
+
     OPENSSL_cleanse(e, sizeof(e));
 }

crypto/engine/eng_lib.c

@@ -12,6 +12,12 @@
 #include <openssl/rand.h>
 #include "internal/refcount.h"
 
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+#include <sanitizer/lsan_interface.h>
+# endif
+#endif
+
 CRYPTO_RWLOCK *global_engine_lock;
 
 CRYPTO_ONCE engine_lock_init = CRYPTO_ONCE_STATIC_INIT;
@@ -33,8 +39,18 @@ ENGINE *ENGINE_new(void)
         ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
         return 0;
     }
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+        __lsan_disable();
+# endif
+#endif
     if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL)
         return NULL;
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+        __lsan_enable();
+# endif
+#endif
     if (!CRYPTO_NEW_REF(&ret->struct_ref, 1)) {
         OPENSSL_free(ret);
         return NULL;

crypto/err/err.c

@@ -26,6 +26,12 @@
 #include "internal/e_os.h"
 #include "err_local.h"
 
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+#include <sanitizer/lsan_interface.h>
+# endif
+#endif
+
 /* Forward declaration in case it's not published because of configuration */
 ERR_STATE *ERR_get_state(void);
 
@@ -689,7 +695,17 @@ ERR_STATE *ossl_err_get_state_int(void)
         if (!CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE *)-1))
             return NULL;
 
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+        __lsan_disable();
+# endif
+#endif
         state = OSSL_ERR_STATE_new();
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+        __lsan_enable();
+# endif
+#endif
         if (state == NULL) {
             CRYPTO_THREAD_set_local(&err_thread_local, NULL);
             return NULL;

crypto/rand/rand_lib.c

@@ -103,6 +103,13 @@ static RAND_GLOBAL *rand_get_global(OSSL_LIB_CTX *libctx)
     return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DRBG_INDEX);
 }
 
+
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+#include <sanitizer/lsan_interface.h>
+# endif
+#endif
+
 #ifndef FIPS_MODULE
 #include <stdio.h>
 #include <time.h>
@@ -857,8 +864,18 @@ static EVP_RAND_CTX *rand_get0_public(OSSL_LIB_CTX *ctx, RAND_GLOBAL *dgbl)
         if (CRYPTO_THREAD_get_local(&dgbl->private) == NULL
             && !ossl_init_thread_start(NULL, ctx, rand_delete_thread_state))
             return NULL;
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+        __lsan_disable();
+# endif
+#endif
         rand = rand_new_drbg(ctx, primary, SECONDARY_RESEED_INTERVAL,
             SECONDARY_RESEED_TIME_INTERVAL);
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+        __lsan_enable();
+# endif
+#endif
         if (!CRYPTO_THREAD_set_local(&dgbl->public, rand)) {
             EVP_RAND_CTX_free(rand);
             rand = NULL;
@@ -899,8 +916,18 @@ static EVP_RAND_CTX *rand_get0_private(OSSL_LIB_CTX *ctx, RAND_GLOBAL *dgbl)
         if (CRYPTO_THREAD_get_local(&dgbl->public) == NULL
             && !ossl_init_thread_start(NULL, ctx, rand_delete_thread_state))
             return NULL;
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+        __lsan_disable();
+# endif
+#endif
         rand = rand_new_drbg(ctx, primary, SECONDARY_RESEED_INTERVAL,
             SECONDARY_RESEED_TIME_INTERVAL);
+#if defined(__has_feature)
+# if __has_feature(address_sanitizer)
+        __lsan_enable();
+# endif
+#endif
         if (!CRYPTO_THREAD_set_local(&dgbl->private, rand)) {
             EVP_RAND_CTX_free(rand);
             rand = NULL;

crypto/sha/sha3.c

@@ -13,6 +13,12 @@
 #endif
 #include "internal/sha3.h"
 
+#if defined(__has_feature)
+#  if __has_feature(memory_sanitizer)
+#    include <sanitizer/msan_interface.h>
+#  endif
+#endif
+
 void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r, int next);
 
 void ossl_sha3_reset(KECCAK1600_CTX *ctx)
@@ -122,8 +128,21 @@ int ossl_sha3_final(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen)
 
     (void)SHA3_absorb(ctx->A, ctx->buf, bsz, bsz);
 
+#if defined(__has_feature)
+#  if __has_feature(memory_sanitizer)
+    __msan_unpoison(ctx->buf, bsz);
+#  endif
+#endif
+
     ctx->xof_state = XOF_STATE_FINAL;
     SHA3_squeeze(ctx->A, out, outlen, bsz, 0);
+
+#if defined(__has_feature)
+#  if __has_feature(memory_sanitizer)
+    __msan_unpoison(out, outlen);
+#  endif
+#endif
+
     return 1;
 }
 
@@ -191,13 +210,27 @@ int ossl_sha3_squeeze(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen)
     if (outlen >= bsz) {
         len = bsz * (outlen / bsz);
         SHA3_squeeze(ctx->A, out, len, bsz, next);
+
+#if defined(__has_feature)
+#  if __has_feature(memory_sanitizer)
+    __msan_unpoison(out, len);
+#  endif
+#endif
+
         next = 1;
         out += len;
         outlen -= len;
     }
     if (outlen > 0) {
         /* Step 3. Squeeze one more block into a buffer */
         SHA3_squeeze(ctx->A, ctx->buf, bsz, bsz, next);
+
+#if defined(__has_feature)
+#  if __has_feature(memory_sanitizer)
+    __msan_unpoison(ctx->buf, bsz);
+#  endif
+#endif
+
         memcpy(out, ctx->buf, outlen);
         /* Step 4. Remember the leftover part of the squeezed block */
         ctx->bufsz = bsz - outlen;

include/internal/refcount.h

@@ -21,8 +21,9 @@
 #define HAVE_C11_ATOMICS
 #endif
 
-#if defined(HAVE_C11_ATOMICS) && defined(ATOMIC_INT_LOCK_FREE) \
-    && ATOMIC_INT_LOCK_FREE > 0
+#  if defined(HAVE_C11_ATOMICS) && defined(ATOMIC_INT_LOCK_FREE) \
+      && ATOMIC_INT_LOCK_FREE > 0 \
+      && 0 /// ClickHouse-specific patch: if we use atomics, tsan complains :(
 
 #define HAVE_ATOMICS 1
 
@@ -74,7 +75,8 @@ static inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
     return 1;
 }
 
-#elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0
+#  elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0 \
+      && 0 /// ClickHouse-specific patch: if we use atomics, tsan complains :(
 
 #define HAVE_ATOMICS 1
 

providers/implementations/rands/drbg_ctr.c

@@ -25,6 +25,12 @@
 #include "internal/provider.h"
 #include "internal/common.h"
 
+#if defined(__has_feature)
+# if __has_feature(memory_sanitizer)
+#  include <sanitizer/msan_interface.h>
+# endif
+#endif
+
 static OSSL_FUNC_rand_newctx_fn drbg_ctr_new_wrapper;
 static OSSL_FUNC_rand_freectx_fn drbg_ctr_free;
 static OSSL_FUNC_rand_instantiate_fn drbg_ctr_instantiate_wrapper;
@@ -72,6 +78,12 @@ static void inc_128(PROV_DRBG_CTR *ctr)
         p[n] = (u8)c;
         c >>= 8;
     } while (n);
+
+#if defined(__has_feature)
+# if __has_feature(memory_sanitizer)
+    __msan_unpoison(p, 16);
+# endif
+#endif
 }
 
 static void ctr_XOR(PROV_DRBG_CTR *ctr, const unsigned char *in, size_t inlen)