From 9f40f7b9ca20547a5e582b87f5ded116b97b9990 Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Tue, 29 Jul 2025 10:04:36 -0700 Subject: [PATCH 1/4] Track the pid on the rb event queue --- src/ruby/ext/grpc/rb_event_thread.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ruby/ext/grpc/rb_event_thread.c b/src/ruby/ext/grpc/rb_event_thread.c index 014dd2dcd8882..7be29d6b374db 100644 --- a/src/ruby/ext/grpc/rb_event_thread.c +++ b/src/ruby/ext/grpc/rb_event_thread.c @@ -36,6 +36,7 @@ typedef struct grpc_rb_event { void* argument; struct grpc_rb_event* next; + pid_t pid; } grpc_rb_event; typedef struct grpc_rb_event_queue { @@ -57,6 +58,7 @@ void grpc_rb_event_queue_enqueue(void (*callback)(void*), void* argument) { grpc_rb_event* event = gpr_malloc(sizeof(grpc_rb_event)); event->callback = callback; event->argument = argument; + event->pid = getpid(); event->next = NULL; gpr_mu_lock(&event_queue.mu); if (event_queue.tail == NULL) { @@ -81,6 +83,12 @@ static grpc_rb_event* grpc_rb_event_queue_dequeue() { event_queue.head = event_queue.head->next; } } + if (event != NULL) { + fprintf(stderr, "DEQUEUED EVENT PID %d %s CURRENT PID %d\n", event->pid, (event->pid == getpid() ? "==" : "!="), getpid()); + } + else { + fprintf(stderr, "DEQUEUED EVENT IS NULL\n"); + } return event; } From 907a5937b681856de462820b643a0f3e1c813878 Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Mon, 4 Aug 2025 12:23:19 -0700 Subject: [PATCH 2/4] Add ruby_backtrace field to grpc_rb_event for debugging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the grpc_rb_event struct to capture Ruby call stack information at event enqueue time. This helps debug cross-process event handling by showing where events originated in the Ruby code. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- src/ruby/ext/grpc/rb_event_thread.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/ruby/ext/grpc/rb_event_thread.c b/src/ruby/ext/grpc/rb_event_thread.c index 7be29d6b374db..ed9a2bb6d6dfe 100644 --- a/src/ruby/ext/grpc/rb_event_thread.c +++ b/src/ruby/ext/grpc/rb_event_thread.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "rb_grpc.h" #include "rb_grpc_imports.generated.h" @@ -37,6 +38,7 @@ typedef struct grpc_rb_event { struct grpc_rb_event* next; pid_t pid; + char* ruby_backtrace; } grpc_rb_event; typedef struct grpc_rb_event_queue { @@ -59,6 +61,19 @@ void grpc_rb_event_queue_enqueue(void (*callback)(void*), void* argument) { event->callback = callback; event->argument = argument; event->pid = getpid(); + + // Capture backtrace using Ruby's caller functionality + VALUE caller_array = rb_funcall(rb_mKernel, rb_intern("caller"), 0); + if (caller_array != Qnil && RARRAY_LEN(caller_array) > 0) { + VALUE backtrace_str = rb_funcall(caller_array, rb_intern("join"), 1, rb_str_new2("\n")); + const char* bt_cstr = StringValueCStr(backtrace_str); + event->ruby_backtrace = gpr_malloc(strlen(bt_cstr) + 1); + strcpy(event->ruby_backtrace, bt_cstr); + } else { + event->ruby_backtrace = gpr_malloc(strlen("no backtrace available") + 1); + strcpy(event->ruby_backtrace, "no backtrace available"); + } + event->next = NULL; gpr_mu_lock(&event_queue.mu); if (event_queue.tail == NULL) { @@ -84,7 +99,8 @@ static grpc_rb_event* grpc_rb_event_queue_dequeue() { } } if (event != NULL) { - fprintf(stderr, "DEQUEUED EVENT PID %d %s CURRENT PID %d\n", event->pid, (event->pid == getpid() ? "==" : "!="), getpid()); + fprintf(stderr, "DEQUEUED EVENT PID %d %s CURRENT PID %d\nBACKTRACE:\n%s\n", + event->pid, (event->pid == getpid() ? "==" : "!="), getpid(), event->ruby_backtrace); } else { fprintf(stderr, "DEQUEUED EVENT IS NULL\n"); @@ -140,6 +156,7 @@ static VALUE grpc_rb_event_thread(void* arg) { break; } else { event->callback(event->argument); + gpr_free(event->ruby_backtrace); gpr_free(event); } } From 38ede99978393205520b70b208012a63be24d0ff Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Mon, 4 Aug 2025 12:31:25 -0700 Subject: [PATCH 3/4] Add c_backtrace field to grpc_rb_event for C stack traces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the grpc_rb_event struct to capture both Ruby and C call stacks for comprehensive debugging. Uses execinfo.h backtrace functions on glibc systems to capture native C stack traces alongside the existing Ruby backtrace functionality. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- src/ruby/ext/grpc/rb_event_thread.c | 36 +++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/ruby/ext/grpc/rb_event_thread.c b/src/ruby/ext/grpc/rb_event_thread.c index ed9a2bb6d6dfe..86f908febb1b4 100644 --- a/src/ruby/ext/grpc/rb_event_thread.c +++ b/src/ruby/ext/grpc/rb_event_thread.c @@ -27,6 +27,9 @@ #include #include #include +#ifdef __GLIBC__ +#include +#endif #include "rb_grpc.h" #include "rb_grpc_imports.generated.h" @@ -39,6 +42,7 @@ typedef struct grpc_rb_event { struct grpc_rb_event* next; pid_t pid; char* ruby_backtrace; + char* c_backtrace; } grpc_rb_event; typedef struct grpc_rb_event_queue { @@ -74,6 +78,32 @@ void grpc_rb_event_queue_enqueue(void (*callback)(void*), void* argument) { strcpy(event->ruby_backtrace, "no backtrace available"); } + // Capture C backtrace +#ifdef __GLIBC__ + void* buffer[256]; + int nptrs = backtrace(buffer, 256); + char** strings = backtrace_symbols(buffer, nptrs); + if (strings != NULL) { + size_t total_len = 0; + for (int i = 0; i < nptrs; i++) { + total_len += strlen(strings[i]) + 1; // +1 for newline + } + event->c_backtrace = gpr_malloc(total_len + 1); // +1 for null terminator + event->c_backtrace[0] = '\0'; + for (int i = 0; i < nptrs; i++) { + strcat(event->c_backtrace, strings[i]); + if (i < nptrs - 1) strcat(event->c_backtrace, "\n"); + } + free(strings); + } else { + event->c_backtrace = gpr_malloc(strlen("no C backtrace available") + 1); + strcpy(event->c_backtrace, "no C backtrace available"); + } +#else + event->c_backtrace = gpr_malloc(strlen("C backtrace not supported on this platform") + 1); + strcpy(event->c_backtrace, "C backtrace not supported on this platform"); +#endif + event->next = NULL; gpr_mu_lock(&event_queue.mu); if (event_queue.tail == NULL) { @@ -99,8 +129,9 @@ static grpc_rb_event* grpc_rb_event_queue_dequeue() { } } if (event != NULL) { - fprintf(stderr, "DEQUEUED EVENT PID %d %s CURRENT PID %d\nBACKTRACE:\n%s\n", - event->pid, (event->pid == getpid() ? "==" : "!="), getpid(), event->ruby_backtrace); + fprintf(stderr, "DEQUEUED EVENT PID %d %s CURRENT PID %d\nRUBY BACKTRACE:\n%s\nC BACKTRACE:\n%s\n", + event->pid, (event->pid == getpid() ? "==" : "!="), getpid(), + event->ruby_backtrace, event->c_backtrace); } else { fprintf(stderr, "DEQUEUED EVENT IS NULL\n"); @@ -157,6 +188,7 @@ static VALUE grpc_rb_event_thread(void* arg) { } else { event->callback(event->argument); gpr_free(event->ruby_backtrace); + gpr_free(event->c_backtrace); gpr_free(event); } } From 5fe03ca17eed7bd0e0f1101ffadd8412ad732b11 Mon Sep 17 00:00:00 2001 From: rwstauner <142719+rwstauner@users.noreply.github.com> Date: Mon, 4 Aug 2025 19:48:16 +0000 Subject: [PATCH 4/4] Automated change: Fix sanity tests --- src/ruby/ext/grpc/rb_event_thread.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/ruby/ext/grpc/rb_event_thread.c b/src/ruby/ext/grpc/rb_event_thread.c index 86f908febb1b4..42c7a6d31f9a3 100644 --- a/src/ruby/ext/grpc/rb_event_thread.c +++ b/src/ruby/ext/grpc/rb_event_thread.c @@ -69,7 +69,8 @@ void grpc_rb_event_queue_enqueue(void (*callback)(void*), void* argument) { // Capture backtrace using Ruby's caller functionality VALUE caller_array = rb_funcall(rb_mKernel, rb_intern("caller"), 0); if (caller_array != Qnil && RARRAY_LEN(caller_array) > 0) { - VALUE backtrace_str = rb_funcall(caller_array, rb_intern("join"), 1, rb_str_new2("\n")); + VALUE backtrace_str = + rb_funcall(caller_array, rb_intern("join"), 1, rb_str_new2("\n")); const char* bt_cstr = StringValueCStr(backtrace_str); event->ruby_backtrace = gpr_malloc(strlen(bt_cstr) + 1); strcpy(event->ruby_backtrace, bt_cstr); @@ -86,9 +87,9 @@ void grpc_rb_event_queue_enqueue(void (*callback)(void*), void* argument) { if (strings != NULL) { size_t total_len = 0; for (int i = 0; i < nptrs; i++) { - total_len += strlen(strings[i]) + 1; // +1 for newline + total_len += strlen(strings[i]) + 1; // +1 for newline } - event->c_backtrace = gpr_malloc(total_len + 1); // +1 for null terminator + event->c_backtrace = gpr_malloc(total_len + 1); // +1 for null terminator event->c_backtrace[0] = '\0'; for (int i = 0; i < nptrs; i++) { strcat(event->c_backtrace, strings[i]); @@ -100,7 +101,8 @@ void grpc_rb_event_queue_enqueue(void (*callback)(void*), void* argument) { strcpy(event->c_backtrace, "no C backtrace available"); } #else - event->c_backtrace = gpr_malloc(strlen("C backtrace not supported on this platform") + 1); + event->c_backtrace = + gpr_malloc(strlen("C backtrace not supported on this platform") + 1); strcpy(event->c_backtrace, "C backtrace not supported on this platform"); #endif @@ -129,11 +131,12 @@ static grpc_rb_event* grpc_rb_event_queue_dequeue() { } } if (event != NULL) { - fprintf(stderr, "DEQUEUED EVENT PID %d %s CURRENT PID %d\nRUBY BACKTRACE:\n%s\nC BACKTRACE:\n%s\n", - event->pid, (event->pid == getpid() ? "==" : "!="), getpid(), + fprintf(stderr, + "DEQUEUED EVENT PID %d %s CURRENT PID %d\nRUBY BACKTRACE:\n%s\nC " + "BACKTRACE:\n%s\n", + event->pid, (event->pid == getpid() ? "==" : "!="), getpid(), event->ruby_backtrace, event->c_backtrace); - } - else { + } else { fprintf(stderr, "DEQUEUED EVENT IS NULL\n"); } return event;