diff --git a/.gitmodules b/.gitmodules index b4c756a67b179..e69de29bb2d1d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,37 +0,0 @@ -[submodule "pyca.cryptography"] - path = pyca-cryptography - url = https://github.com/pyca/cryptography.git - -[submodule "krb5"] - path = krb5 - url = https://github.com/krb5/krb5 - -[submodule "gost-engine"] - path = gost-engine - url = https://github.com/gost-engine/engine - update = rebase -[submodule "wycheproof"] - path = wycheproof - url = https://github.com/google/wycheproof -[submodule "tlsfuzzer"] - path = tlsfuzzer - url = https://github.com/tlsfuzzer/tlsfuzzer -[submodule "python-ecdsa"] - path = python-ecdsa - url = https://github.com/tlsfuzzer/python-ecdsa -[submodule "tlslite-ng"] - path = tlslite-ng - url = https://github.com/tlsfuzzer/tlslite-ng -[submodule "oqs-provider"] - path = oqs-provider - url = https://github.com/open-quantum-safe/oqs-provider.git -[submodule "cloudflare-quiche"] - path = cloudflare-quiche - url = https://github.com/cloudflare/quiche -[submodule "fuzz/corpora"] - path = fuzz/corpora - url = https://github.com/openssl/fuzz-corpora - branch = main -[submodule "pkcs11-provider"] - path = pkcs11-provider - url = https://github.com/latchset/pkcs11-provider.git diff --git a/cloudflare-quiche b/cloudflare-quiche deleted file mode 160000 index 7ab6a55cfe471..0000000000000 --- a/cloudflare-quiche +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ab6a55cfe471267d61e4d28ba43d41defcd87e0 diff --git a/crypto/armcap.c b/crypto/armcap.c index 3005d295cfd0b..415c55cb46fc9 100644 --- a/crypto/armcap.c +++ b/crypto/armcap.c @@ -77,6 +77,10 @@ void OPENSSL_cpuid_setup(void) __attribute__((constructor)); #include #define OSSL_IMPLEMENT_GETAUXVAL #endif +#elif defined(__MUSL__) +/* musl has always provided getauxval(), unversioned. */ +#include +#define OSSL_IMPLEMENT_GETAUXVAL #endif #if defined(__FreeBSD__) || defined(__OpenBSD__) #include diff --git a/crypto/bio/bss_dgram.c b/crypto/bio/bss_dgram.c index 968104a062086..8d802bb020a45 100644 --- a/crypto/bio/bss_dgram.c +++ b/crypto/bio/bss_dgram.c @@ -49,7 +49,9 @@ #define M_METHOD_WSARECVMSG 4 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ) -#if !(__GLIBC_PREREQ(2, 14)) +/// ClickHouse-specific patch: Pretend to use a stone age glibc because we use a stone age glibc. +/// Otherwise, system calls sendmmsg and recvmmsg are used which work only with too-new glibc 2.14. +/// #if !(__GLIBC_PREREQ(2, 14)) #undef NO_RECVMMSG /* * Some old glibc versions may have recvmmsg and MSG_WAITFORONE flag, but @@ -57,7 +59,7 @@ * versions */ #define NO_RECVMMSG -#endif +/// #endif #endif #if defined(__GNU__) /* GNU/Hurd does not have IP_PKTINFO yet */ diff --git a/crypto/bn/bn_intern.c b/crypto/bn/bn_intern.c index bd299cd1442d7..6fa404afce2cd 100644 --- a/crypto/bn/bn_intern.c +++ b/crypto/bn/bn_intern.c @@ -10,6 +10,11 @@ #include "internal/cryptlib.h" #include "bn_local.h" +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include +# endif +#endif /* * Determine the modified width-(w+1) Non-Adjacent Form (wNAF) of 'scalar'. * This is an array r[] of values that are either zero or odd with an @@ -188,6 +193,11 @@ int bn_set_words(BIGNUM *a, const BN_ULONG *words, int num_words) return 0; } +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + __msan_unpoison(words, sizeof(BN_ULONG) * num_words); +# endif +#endif memcpy(a->d, words, sizeof(BN_ULONG) * num_words); a->top = num_words; bn_correct_top(a); diff --git a/crypto/cpuid.c b/crypto/cpuid.c index d659135919d1f..18c648148a30c 100644 --- a/crypto/cpuid.c +++ b/crypto/cpuid.c @@ -102,7 +102,16 @@ void OPENSSL_cpuid_setup(void) if (trigger) return; + /// This function is called from .init section before memory sanitizer mmaps shadow memory. + /// Program will crash with segmentation fault when trying access `trigger`, + /// because its address was replaced with some not mapped address. + /// Also see https://github.com/ClickHouse/openssl/pull/5 + /// Unfortunately, __msan_init() is no longer part of msan's public header and there seems to be no replacement. +#if defined(__has_feature) +# if !__has_feature(memory_sanitizer) trigger = 1; +# endif +#endif if ((env = ossl_getenv("OPENSSL_ia32cap")) != NULL) { int off = (env[0] == '~') ? 1 : 0; diff --git a/crypto/ec/curve25519.c b/crypto/ec/curve25519.c index c6886763aba7e..5661d33485832 100644 --- a/crypto/ec/curve25519.c +++ b/crypto/ec/curve25519.c @@ -21,6 +21,12 @@ #include "internal/numbers.h" +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include +# endif +#endif + #if defined(X25519_ASM) && (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) #define BASE_2_64_IMPLEMENTED @@ -5846,6 +5852,12 @@ int ossl_x25519(uint8_t out_shared_key[32], const uint8_t private_key[32], { static const uint8_t kZeros[32] = { 0 }; x25519_scalar_mult(out_shared_key, private_key, peer_public_value); +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + /* x25519_scalar_mult may use assembly that MSan cannot instrument. */ + __msan_unpoison(out_shared_key, 32); +# endif +#endif /* The all-zero output results when the input is a point of small order. */ return CRYPTO_memcmp(kZeros, out_shared_key, 32) != 0; } @@ -5875,5 +5887,11 @@ void ossl_x25519_public_from_private(uint8_t out_public_value[32], fe_mul(zplusy, zplusy, zminusy_inv); fe_tobytes(out_public_value, zplusy); +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + __msan_unpoison(out_public_value, 32); +# endif +#endif + OPENSSL_cleanse(e, sizeof(e)); } diff --git a/crypto/engine/eng_lib.c b/crypto/engine/eng_lib.c index 04a2602ed2d8c..bc643a1b4e67f 100644 --- a/crypto/engine/eng_lib.c +++ b/crypto/engine/eng_lib.c @@ -12,6 +12,12 @@ #include #include "internal/refcount.h" +#if defined(__has_feature) +# if __has_feature(address_sanitizer) +#include +# endif +#endif + CRYPTO_RWLOCK *global_engine_lock; CRYPTO_ONCE engine_lock_init = CRYPTO_ONCE_STATIC_INIT; @@ -33,8 +39,18 @@ ENGINE *ENGINE_new(void) ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB); return 0; } +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + __lsan_disable(); +# endif +#endif if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) return NULL; +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + __lsan_enable(); +# endif +#endif if (!CRYPTO_NEW_REF(&ret->struct_ref, 1)) { OPENSSL_free(ret); return NULL; diff --git a/crypto/err/err.c b/crypto/err/err.c index a995c4e2422de..2a0af875b44d2 100644 --- a/crypto/err/err.c +++ b/crypto/err/err.c @@ -26,6 +26,12 @@ #include "internal/e_os.h" #include "err_local.h" +#if defined(__has_feature) +# if __has_feature(address_sanitizer) +#include +# endif +#endif + /* Forward declaration in case it's not published because of configuration */ ERR_STATE *ERR_get_state(void); @@ -689,7 +695,17 @@ ERR_STATE *ossl_err_get_state_int(void) if (!CRYPTO_THREAD_set_local(&err_thread_local, (ERR_STATE *)-1)) return NULL; +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + __lsan_disable(); +# endif +#endif state = OSSL_ERR_STATE_new(); +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + __lsan_enable(); +# endif +#endif if (state == NULL) { CRYPTO_THREAD_set_local(&err_thread_local, NULL); return NULL; diff --git a/crypto/rand/rand_lib.c b/crypto/rand/rand_lib.c index df70d1c2b1dad..893cadd22bb79 100644 --- a/crypto/rand/rand_lib.c +++ b/crypto/rand/rand_lib.c @@ -103,6 +103,13 @@ static RAND_GLOBAL *rand_get_global(OSSL_LIB_CTX *libctx) return ossl_lib_ctx_get_data(libctx, OSSL_LIB_CTX_DRBG_INDEX); } + +#if defined(__has_feature) +# if __has_feature(address_sanitizer) +#include +# endif +#endif + #ifndef FIPS_MODULE #include #include @@ -857,8 +864,18 @@ static EVP_RAND_CTX *rand_get0_public(OSSL_LIB_CTX *ctx, RAND_GLOBAL *dgbl) if (CRYPTO_THREAD_get_local(&dgbl->private) == NULL && !ossl_init_thread_start(NULL, ctx, rand_delete_thread_state)) return NULL; +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + __lsan_disable(); +# endif +#endif rand = rand_new_drbg(ctx, primary, SECONDARY_RESEED_INTERVAL, SECONDARY_RESEED_TIME_INTERVAL); +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + __lsan_enable(); +# endif +#endif if (!CRYPTO_THREAD_set_local(&dgbl->public, rand)) { EVP_RAND_CTX_free(rand); rand = NULL; @@ -899,8 +916,18 @@ static EVP_RAND_CTX *rand_get0_private(OSSL_LIB_CTX *ctx, RAND_GLOBAL *dgbl) if (CRYPTO_THREAD_get_local(&dgbl->public) == NULL && !ossl_init_thread_start(NULL, ctx, rand_delete_thread_state)) return NULL; +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + __lsan_disable(); +# endif +#endif rand = rand_new_drbg(ctx, primary, SECONDARY_RESEED_INTERVAL, SECONDARY_RESEED_TIME_INTERVAL); +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + __lsan_enable(); +# endif +#endif if (!CRYPTO_THREAD_set_local(&dgbl->private, rand)) { EVP_RAND_CTX_free(rand); rand = NULL; diff --git a/crypto/sha/sha3.c b/crypto/sha/sha3.c index 21e1070beed27..1cdc8e983e8e8 100644 --- a/crypto/sha/sha3.c +++ b/crypto/sha/sha3.c @@ -13,6 +13,12 @@ #endif #include "internal/sha3.h" +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include +# endif +#endif + void SHA3_squeeze(uint64_t A[5][5], unsigned char *out, size_t len, size_t r, int next); void ossl_sha3_reset(KECCAK1600_CTX *ctx) @@ -122,8 +128,21 @@ int ossl_sha3_final(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen) (void)SHA3_absorb(ctx->A, ctx->buf, bsz, bsz); +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + __msan_unpoison(ctx->buf, bsz); +# endif +#endif + ctx->xof_state = XOF_STATE_FINAL; SHA3_squeeze(ctx->A, out, outlen, bsz, 0); + +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + __msan_unpoison(out, outlen); +# endif +#endif + return 1; } @@ -191,6 +210,13 @@ int ossl_sha3_squeeze(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen) if (outlen >= bsz) { len = bsz * (outlen / bsz); SHA3_squeeze(ctx->A, out, len, bsz, next); + +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + __msan_unpoison(out, len); +# endif +#endif + next = 1; out += len; outlen -= len; @@ -198,6 +224,13 @@ int ossl_sha3_squeeze(KECCAK1600_CTX *ctx, unsigned char *out, size_t outlen) if (outlen > 0) { /* Step 3. Squeeze one more block into a buffer */ SHA3_squeeze(ctx->A, ctx->buf, bsz, bsz, next); + +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + __msan_unpoison(ctx->buf, bsz); +# endif +#endif + memcpy(out, ctx->buf, outlen); /* Step 4. Remember the leftover part of the squeezed block */ ctx->bufsz = bsz - outlen; diff --git a/fuzz/corpora b/fuzz/corpora deleted file mode 160000 index ce771805c094d..0000000000000 --- a/fuzz/corpora +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ce771805c094d098c25a218bc8e9f7344eccbc5a diff --git a/gost-engine b/gost-engine deleted file mode 160000 index 74b1f4fddbc2d..0000000000000 --- a/gost-engine +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 74b1f4fddbc2d6de969815b1992ddc1ae7c643fe diff --git a/include/internal/refcount.h b/include/internal/refcount.h index 61eb78ae41205..5c11c80fc15fa 100644 --- a/include/internal/refcount.h +++ b/include/internal/refcount.h @@ -21,8 +21,9 @@ #define HAVE_C11_ATOMICS #endif -#if defined(HAVE_C11_ATOMICS) && defined(ATOMIC_INT_LOCK_FREE) \ - && ATOMIC_INT_LOCK_FREE > 0 +# if defined(HAVE_C11_ATOMICS) && defined(ATOMIC_INT_LOCK_FREE) \ + && ATOMIC_INT_LOCK_FREE > 0 \ + && 0 /// ClickHouse-specific patch: if we use atomics, tsan complains :( #define HAVE_ATOMICS 1 @@ -74,7 +75,8 @@ static inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret) return 1; } -#elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0 +# elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0 \ + && 0 /// ClickHouse-specific patch: if we use atomics, tsan complains :( #define HAVE_ATOMICS 1 diff --git a/krb5 b/krb5 deleted file mode 160000 index 784c38f50e70a..0000000000000 --- a/krb5 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 784c38f50e70a739400cdd3f2620bac2e2788e6c diff --git a/oqs-provider b/oqs-provider deleted file mode 160000 index 7bc597c04b534..0000000000000 --- a/oqs-provider +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7bc597c04b534ddea9b6654481deb31ded8e1bbc diff --git a/pkcs11-provider b/pkcs11-provider deleted file mode 160000 index 64fc325ac0f91..0000000000000 --- a/pkcs11-provider +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 64fc325ac0f91d03d76b3546df2998d3a38c525b diff --git a/providers/implementations/rands/drbg_ctr.c b/providers/implementations/rands/drbg_ctr.c index 57da1cfdf33df..fa4a7400ffbd2 100644 --- a/providers/implementations/rands/drbg_ctr.c +++ b/providers/implementations/rands/drbg_ctr.c @@ -25,6 +25,12 @@ #include "internal/provider.h" #include "internal/common.h" +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include +# endif +#endif + static OSSL_FUNC_rand_newctx_fn drbg_ctr_new_wrapper; static OSSL_FUNC_rand_freectx_fn drbg_ctr_free; static OSSL_FUNC_rand_instantiate_fn drbg_ctr_instantiate_wrapper; @@ -72,6 +78,12 @@ static void inc_128(PROV_DRBG_CTR *ctr) p[n] = (u8)c; c >>= 8; } while (n); + +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + __msan_unpoison(p, 16); +# endif +#endif } static void ctr_XOR(PROV_DRBG_CTR *ctr, const unsigned char *in, size_t inlen) diff --git a/providers/implementations/rands/seeding/rand_unix.c b/providers/implementations/rands/seeding/rand_unix.c index 0b8a9ec341d92..ed82495e693fc 100644 --- a/providers/implementations/rands/seeding/rand_unix.c +++ b/providers/implementations/rands/seeding/rand_unix.c @@ -351,8 +351,13 @@ static ssize_t syscall_random(void *buf, size_t buflen) * Note: Sometimes getentropy() can be provided but not implemented * internally. So we need to check errno for ENOSYS */ -#if !defined(__DragonFly__) && !defined(__NetBSD__) && !defined(__FreeBSD__) -#if defined(__GNUC__) && __GNUC__ >= 2 && defined(__ELF__) && !defined(__hpux) +# if !defined(__DragonFly__) && !defined(__NetBSD__) && !defined(__FreeBSD__) + + /// Disable the usage of "getentropy" function from libc (on static link time) to avoid dependency on too new libc version. + /// Otherwise, if we build ClickHouse on a system with new libc and run the built binary on a system with old libc, it will fail. + /// + /// Note that there is a fallback below to (1) runtime symbol lookup and (2) direct syscall, that are equivalent. +# if 0 && defined(__GNUC__) && __GNUC__>=2 && defined(__ELF__) && !defined(__hpux) extern int getentropy(void *buffer, size_t length) __attribute__((weak)); if (getentropy != NULL) { diff --git a/pyca-cryptography b/pyca-cryptography deleted file mode 160000 index 7e33b0e7739d6..0000000000000 --- a/pyca-cryptography +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7e33b0e7739d633c77b8c478620167f693ed13f4 diff --git a/python-ecdsa b/python-ecdsa deleted file mode 160000 index 4096fa0171592..0000000000000 --- a/python-ecdsa +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4096fa01715929e08b97e73f3173aee9d57f2a3f diff --git a/tlsfuzzer b/tlsfuzzer deleted file mode 160000 index 61f45d9701294..0000000000000 --- a/tlsfuzzer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 61f45d9701294fd87ef92d2a7e3dfb076653a562 diff --git a/tlslite-ng b/tlslite-ng deleted file mode 160000 index 77ef321dde1a9..0000000000000 --- a/tlslite-ng +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 77ef321dde1a9e6bcf94d73c80f8789a770d8031 diff --git a/wycheproof b/wycheproof deleted file mode 160000 index 2196000605e45..0000000000000 --- a/wycheproof +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2196000605e45d91097147c9c71f26b72af58003