Hello, that is the other failure I found.
When debugging it manually the _syscall_hook_trampoline is only reached in the exit handlers, while in Debian 12 bookworm it stops below __GI___libc_write and _syscall_hook_trampoline_48_3d_00_f0_ff_ff.
So this is maybe a missing syscall_patch_hooks?
I attempted to create a fix by just copying the _syscall_hook_trampoline_c3_nop with the additional pop %rbx.
And the test then stops indeed inside the __GI___libc_write, but the backtrace looks not clean,
and the third finish instruction of the test fails.
So I guess there is some more cfi/cfa magic needed to adjust for the additional pop %rbx.
Debian 13 trixie
(gdb) disassemble /r __GI___libc_write
Dump of assembler code for function __GI___libc_write:
0x00007ffff7e90960 <+0>: 48 83 ec 10 sub $0x10,%rsp
0x00007ffff7e90964 <+4>: 48 63 ff movslq %edi,%rdi
0x00007ffff7e90967 <+7>: 45 31 c9 xor %r9d,%r9d
0x00007ffff7e9096a <+10>: 45 31 c0 xor %r8d,%r8d
0x00007ffff7e9096d <+13>: 6a 01 push $0x1
0x00007ffff7e9096f <+15>: 31 c9 xor %ecx,%ecx
0x00007ffff7e90971 <+17>: e8 2a ad f8 ff call 0x7ffff7e1b6a0 <__syscall_cancel>
0x00007ffff7e90976 <+22>: 48 83 c4 18 add $0x18,%rsp
0x00007ffff7e9097a <+26>: c3 ret
End of assembler dump.
(gdb) disassemble /r __syscall_cancel
Dump of assembler code for function __syscall_cancel:
0x00007ffff7e1b6a0 <+0>: 48 83 ec 10 sub $0x10,%rsp
0x00007ffff7e1b6a4 <+4>: ff 74 24 18 push 0x18(%rsp)
0x00007ffff7e1b6a8 <+8>: e8 73 ff ff ff call 0x7ffff7e1b620 <__internal_syscall_cancel>
0x00007ffff7e1b6ad <+13>: 5a pop %rdx
0x00007ffff7e1b6ae <+14>: 59 pop %rcx
0x00007ffff7e1b6af <+15>: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax
0x00007ffff7e1b6b5 <+21>: 77 09 ja 0x7ffff7e1b6c0 <__syscall_cancel+32>
0x00007ffff7e1b6b7 <+23>: 48 83 c4 08 add $0x8,%rsp
0x00007ffff7e1b6bb <+27>: c3 ret
0x00007ffff7e1b6bc <+28>: 0f 1f 40 00 nopl 0x0(%rax)
0x00007ffff7e1b6c0 <+32>: 48 8b 15 29 57 15 00 mov 0x155729(%rip),%rdx # 0x7ffff7f70df0
0x00007ffff7e1b6c7 <+39>: f7 d8 neg %eax
0x00007ffff7e1b6c9 <+41>: 64 89 02 mov %eax,%fs:(%rdx)
0x00007ffff7e1b6cc <+44>: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax
0x00007ffff7e1b6d3 <+51>: 48 83 c4 08 add $0x8,%rsp
0x00007ffff7e1b6d7 <+55>: c3 ret
End of assembler dump.
(gdb) disassemble /r __internal_syscall_cancel
Dump of assembler code for function __internal_syscall_cancel:
0x00007ffff7e1b620 <+0>: 53 push %rbx
0x00007ffff7e1b621 <+1>: 49 89 ca mov %rcx,%r10
0x00007ffff7e1b624 <+4>: 64 48 8b 1c 25 10 00 00 00 mov %fs:0x10,%rbx
0x00007ffff7e1b62d <+13>: 8b 83 08 03 00 00 mov 0x308(%rbx),%eax
0x00007ffff7e1b633 <+19>: 80 3d c6 da 15 00 00 cmpb $0x0,0x15dac6(%rip) # 0x7ffff7f79100 <__libc_single_threaded>
0x00007ffff7e1b63a <+26>: 75 44 jne 0x7ffff7e1b680 <__internal_syscall_cancel+96>
0x00007ffff7e1b63c <+28>: a8 01 test $0x1,%al
0x00007ffff7e1b63e <+30>: 75 40 jne 0x7ffff7e1b680 <__internal_syscall_cancel+96>
0x00007ffff7e1b640 <+32>: a8 10 test $0x10,%al
0x00007ffff7e1b642 <+34>: 75 3c jne 0x7ffff7e1b680 <__internal_syscall_cancel+96>
0x00007ffff7e1b644 <+36>: 41 51 push %r9
0x00007ffff7e1b646 <+38>: 4c 8d 9b 08 03 00 00 lea 0x308(%rbx),%r11
0x00007ffff7e1b64d <+45>: 49 89 c9 mov %rcx,%r9
0x00007ffff7e1b650 <+48>: 48 89 f1 mov %rsi,%rcx
0x00007ffff7e1b653 <+51>: 41 50 push %r8
0x00007ffff7e1b655 <+53>: 48 8b 74 24 20 mov 0x20(%rsp),%rsi
0x00007ffff7e1b65a <+58>: 49 89 d0 mov %rdx,%r8
0x00007ffff7e1b65d <+61>: 48 89 fa mov %rdi,%rdx
0x00007ffff7e1b660 <+64>: 4c 89 df mov %r11,%rdi
0x00007ffff7e1b663 <+67>: e8 58 b3 00 00 call 0x7ffff7e269c0 <__syscall_cancel_arch>
0x00007ffff7e1b668 <+72>: 8b 93 08 03 00 00 mov 0x308(%rbx),%edx
0x00007ffff7e1b66e <+78>: 59 pop %rcx
0x00007ffff7e1b66f <+79>: 5e pop %rsi
0x00007ffff7e1b670 <+80>: 48 83 f8 fc cmp $0xfffffffffffffffc,%rax
0x00007ffff7e1b674 <+84>: 74 1a je 0x7ffff7e1b690 <__internal_syscall_cancel+112>
0x00007ffff7e1b676 <+86>: 5b pop %rbx
0x00007ffff7e1b677 <+87>: c3 ret
0x00007ffff7e1b678 <+88>: 0f 1f 84 00 00 00 00 00 nopl 0x0(%rax,%rax,1)
0x00007ffff7e1b680 <+96>: 48 8b 44 24 10 mov 0x10(%rsp),%rax
=> 0x00007ffff7e1b685 <+101>: 0f 05 syscall
0x00007ffff7e1b687 <+103>: 5b pop %rbx
0x00007ffff7e1b688 <+104>: c3 ret
0x00007ffff7e1b689 <+105>: 0f 1f 80 00 00 00 00 nopl 0x0(%rax)
0x00007ffff7e1b690 <+112>: 83 e2 39 and $0x39,%edx
0x00007ffff7e1b693 <+115>: 83 fa 08 cmp $0x8,%edx
0x00007ffff7e1b696 <+118>: 75 de jne 0x7ffff7e1b676 <__internal_syscall_cancel+86>
0x00007ffff7e1b698 <+120>: e8 23 ff ff ff call 0x7ffff7e1b5c0 <__syscall_do_cancel>
End of assembler dump.
(gdb)
Debian 12 bookworm
(gdb) disassemble /r __GI___libc_write
Dump of assembler code for function __GI___libc_write:
0x00007ffff7ed1330 <+0>: 80 3d a1 32 0e 00 00 cmpb $0x0,0xe32a1(%rip) # 0x7ffff7fb45d8 <__libc_single_threaded>
0x00007ffff7ed1337 <+7>: 74 17 je 0x7ffff7ed1350 <__GI___libc_write+32>
0x00007ffff7ed1339 <+9>: b8 01 00 00 00 mov $0x1,%eax
0x00007ffff7ed133e <+14>: 0f 05 syscall
=> 0x00007ffff7ed1340 <+16>: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax
0x00007ffff7ed1346 <+22>: 77 58 ja 0x7ffff7ed13a0 <__GI___libc_write+112>
0x00007ffff7ed1348 <+24>: c3 ret
0x00007ffff7ed1349 <+25>: 0f 1f 80 00 00 00 00 nopl 0x0(%rax)
0x00007ffff7ed1350 <+32>: 48 83 ec 28 sub $0x28,%rsp
0x00007ffff7ed1354 <+36>: 48 89 54 24 18 mov %rdx,0x18(%rsp)
0x00007ffff7ed1359 <+41>: 48 89 74 24 10 mov %rsi,0x10(%rsp)
0x00007ffff7ed135e <+46>: 89 7c 24 08 mov %edi,0x8(%rsp)
0x00007ffff7ed1362 <+50>: e8 b9 d4 f8 ff call 0x7ffff7e5e820 <__GI___pthread_enable_asynccancel>
0x00007ffff7ed1367 <+55>: 48 8b 54 24 18 mov 0x18(%rsp),%rdx
0x00007ffff7ed136c <+60>: 48 8b 74 24 10 mov 0x10(%rsp),%rsi
0x00007ffff7ed1371 <+65>: 41 89 c0 mov %eax,%r8d
0x00007ffff7ed1374 <+68>: 8b 7c 24 08 mov 0x8(%rsp),%edi
0x00007ffff7ed1378 <+72>: b8 01 00 00 00 mov $0x1,%eax
0x00007ffff7ed137d <+77>: 0f 05 syscall
0x00007ffff7ed137f <+79>: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax
0x00007ffff7ed1385 <+85>: 77 31 ja 0x7ffff7ed13b8 <__GI___libc_write+136>
0x00007ffff7ed1387 <+87>: 44 89 c7 mov %r8d,%edi
0x00007ffff7ed138a <+90>: 48 89 44 24 08 mov %rax,0x8(%rsp)
0x00007ffff7ed138f <+95>: e8 0c d5 f8 ff call 0x7ffff7e5e8a0 <__GI___pthread_disable_asynccancel>
0x00007ffff7ed1394 <+100>: 48 8b 44 24 08 mov 0x8(%rsp),%rax
0x00007ffff7ed1399 <+105>: 48 83 c4 28 add $0x28,%rsp
0x00007ffff7ed139d <+109>: c3 ret
0x00007ffff7ed139e <+110>: 66 90 xchg %ax,%ax
0x00007ffff7ed13a0 <+112>: 48 8b 15 39 aa 0d 00 mov 0xdaa39(%rip),%rdx # 0x7ffff7fabde0
0x00007ffff7ed13a7 <+119>: f7 d8 neg %eax
0x00007ffff7ed13a9 <+121>: 64 89 02 mov %eax,%fs:(%rdx)
0x00007ffff7ed13ac <+124>: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax
0x00007ffff7ed13b3 <+131>: c3 ret
0x00007ffff7ed13b4 <+132>: 0f 1f 40 00 nopl 0x0(%rax)
0x00007ffff7ed13b8 <+136>: 48 8b 15 21 aa 0d 00 mov 0xdaa21(%rip),%rdx # 0x7ffff7fabde0
0x00007ffff7ed13bf <+143>: f7 d8 neg %eax
0x00007ffff7ed13c1 <+145>: 64 89 02 mov %eax,%fs:(%rdx)
0x00007ffff7ed13c4 <+148>: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax
0x00007ffff7ed13cb <+155>: eb ba jmp 0x7ffff7ed1387 <__GI___libc_write+87>
End of assembler dump.
(gdb)
Attempt to create a patch for rr
diff --git a/src/preload/syscall_hook.S b/src/preload/syscall_hook.S
index 15dd12b8..f4810275 100644
--- a/src/preload/syscall_hook.S
+++ b/src/preload/syscall_hook.S
@@ -567,6 +567,27 @@ SYSCALLHOOK_START(_syscall_hook_trampoline_c3_nop)
.cfi_endproc
.size _syscall_hook_trampoline_c3_nop, .-_syscall_hook_trampoline_c3_nop
+SYSCALLHOOK_START(_syscall_hook_trampoline_5b_c3_nop)
+ .cfi_offset %rip, 16
+ RSP_IS_CFA_PLUS_OFFSET(24)
+ callq __morestack
+ /* The original instructions after the syscall are
+ pop %rbx; retq; nopl 0x0(%rax,%rax,1) */
+ /* We're not returning to the dynamically generated stub, so
+ we need to fix the stack pointer ourselves. */
+ pop %rbx
+ CFA_AT_RSP_OFFSET(0)
+ pop %rsp
+ .cfi_def_cfa %rsp, 0;
+ pop %rbx
+ .cfi_adjust_cfa_offset -8
+ pop (stub_scratch_1)
+ .cfi_adjust_cfa_offset -8
+ jmp _syscallbuf_final_exit_instruction
+
+ .cfi_endproc
+ .size _syscall_hook_trampoline_5b_c3_nop, .-_syscall_hook_trampoline_5b_c3_nop
+
SYSCALLHOOK_START(_syscall_hook_trampoline_40_80_f6_81)
xor $0x81, %sil
call __morestack
diff --git a/src/preload/syscallbuf.c b/src/preload/syscallbuf.c
index 2b30d9fc..ab024490 100644
--- a/src/preload/syscallbuf.c
+++ b/src/preload/syscallbuf.c
@@ -818,6 +818,7 @@ static void __attribute__((constructor)) init_process(void) {
extern RR_HIDDEN void _syscall_hook_trampoline_ba_01_00_00_00(void);
extern RR_HIDDEN void _syscall_hook_trampoline_89_c1_31_d2(void);
extern RR_HIDDEN void _syscall_hook_trampoline_c3_nop(void);
+ extern RR_HIDDEN void _syscall_hook_trampoline_5b_c3_nop(void);
extern RR_HIDDEN void _syscall_hook_trampoline_40_80_f6_81(void);
extern RR_HIDDEN void _syscall_hook_trampoline_49_89_ca(void);
extern RR_HIDDEN void _syscall_hook_trampoline_48_89_c1(void);
@@ -973,6 +974,12 @@ static void __attribute__((constructor)) init_process(void) {
3,
{ 0xc3, 0xcc, 0xcc },
(uintptr_t)_syscall_hook_trampoline_c3_nop },
+ /* __internal_syscall_cancel has 'syscall' followed by 'pop %rbx; retq; nopl 0x0(%rax,%rax,1)'
+ in debian-13-trixie libc6 2.41-12+deb13u2 */
+ { PATCH_IS_MULTIPLE_INSTRUCTIONS,
+ 9,
+ { 0x5b, 0xc3, 0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00 },
+ (uintptr_t)_syscall_hook_trampoline_5b_c3_nop },
/* glibc-2.31 on Ubuntu 20.04 has 'xor $0x81, %sil' followed by 'syscall' */
{ PATCH_SYSCALL_INSTRUCTION_IS_LAST,
4,EDIT: slightly improved version (comment above belongs to the element; use in both pop instructions rbx)
Unclean backtrace with above patch
(rr) bt
#0 _syscall_hook_trampoline () at src/preload/syscall_hook.S:256
#1 0x00007fac6f9e834d in __morestack () at src/preload/syscall_hook.S:443
#2 0x00007fac6f9e8488 in _syscall_hook_trampoline_5b_c3_nop () at src/preload/syscall_hook.S:573
#3 0x0000000000000001 in ?? ()
#4 0x00007fac6f7ba600 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x00007fac6f9ff910 in ?? ()
#6 0x00007fac6f8b5976 in __GI___libc_write (fd=<optimized out>, buf=<optimized out>, nbytes=<optimized out>) at ../sysdeps/unix/sysv/linux/write.c:26
#7 0x000055b181eae224 in atomic_printf (fmt=0x55b181eaf008 "%s\n") at src/test/x86/../util.h:161
#8 0x000055b181eae24d in atomic_puts (str=0x55b181eaf033 "EXIT-SUCCESS") at src/test/x86/../util.h:170
#9 0x000055b181eae2d9 in main () at src/test/x86/morestack_unwind.c:21
Hello, that is the other failure I found.
When debugging it manually the
_syscall_hook_trampolineis only reached in the exit handlers, while in Debian 12 bookworm it stops below __GI___libc_write and _syscall_hook_trampoline_48_3d_00_f0_ff_ff.So this is maybe a missing
syscall_patch_hooks?I attempted to create a fix by just copying the
_syscall_hook_trampoline_c3_nopwith the additionalpop %rbx.And the test then stops indeed inside the
__GI___libc_write, but the backtrace looks not clean,and the third finish instruction of the test fails.
So I guess there is some more cfi/cfa magic needed to adjust for the additional
pop %rbx.Debian 13 trixie
Debian 12 bookworm
Attempt to create a patch for rr
EDIT: slightly improved version (comment above belongs to the element; use in both pop instructions rbx)
Unclean backtrace with above patch