M-Labs
/
libfringe
3
0
Fork 2
Code Issues Pull Requests Projects Releases Wiki Activity

Add proper CFI annotations to the inline asm 

This ensures that gdb is able to generate a correct backtrace when
stopped at any instruction in the inline asm.

It also makes backtraces accessible to tools that only track frame
pointer chains, like perf or dtrace.

close #49
This commit is contained in:
Amanieu d'Antras 2016-09-03 12:07:00 +01:00 committed by edef
parent 0a99491f70
commit f7f209c1eb
3 changed files with 337 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
src/arch/or1k.rs
View File

@ -1,6 +1,7 @@
// This file is part of libfringe, a low-level green threading library. // This file is part of libfringe, a low-level green threading library.
// Copyright (c) edef <edef@edef.eu>, // Copyright (c) edef <edef@edef.eu>,
// whitequark <whitequark@whitequark.org> // whitequark <whitequark@whitequark.org>
// Amanieu d'Antras <amanieu@gmail.com>
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be // http://opensource.org/licenses/MIT>, at your option. This file may not be
@ -14,6 +15,31 @@
// while swapping context; this is an arbitrary choice // while swapping context; this is an arbitrary choice
// (we clobber all registers and could use any of them) but this allows us // (we clobber all registers and could use any of them) but this allows us
// to reuse the swap function to perform the initial call. // to reuse the swap function to perform the initial call.
//
// To understand the DWARF CFI code in this file, keep in mind these facts:
// * CFI is "call frame information"; a set of instructions to a debugger or
// an unwinder that allow it to simulate returning from functions. This implies
// restoring every register to its pre-call state, as well as the stack pointer.
// * CFA is "call frame address"; the value of stack pointer right before the call
// instruction in the caller. Everything strictly below CFA (and inclusive until
// the next CFA) is the call frame of the callee. This implies that the return
// address is the part of callee's call frame.
// * Logically, DWARF CFI is a table where rows are instruction pointer values and
// columns describe where registers are spilled (mostly using expressions that
// compute a memory location as CFA+n). A .cfi_offset pseudoinstruction changes
// the state of a column for all IP numerically larger than the one it's placed
// after. A .cfi_def_* pseudoinstruction changes the CFA value similarly.
// * Simulating return is as easy as restoring register values from the CFI table
// and then setting stack pointer to CFA.
//
// A high-level overview of the function of the trampolines when unwinding is:
// * The 2nd init trampoline puts a controlled value (written in swap to `new_cfa`)
// into r2. This is then used as the CFA for the 1st trampoline.
// * This controlled value points to the bottom of the stack of the parent context,
// which holds the saved r2 and r9 from the call to swap().
// * The 1st init trampoline tells the unwinder to restore r2 and r9
// from the stack frame at r2 (in the parent stack), thus continuing
// unwinding at the swap call site instead of falling off the end of context stack.
use stack::Stack; use stack::Stack;
pub const STACK_ALIGNMENT: usize = 4; pub const STACK_ALIGNMENT: usize = 4;
@ -23,7 +49,7 @@ pub struct StackPointer(*mut usize);
pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackPointer { pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackPointer {
#[naked] #[naked]
unsafe extern "C" fn trampoline() { unsafe extern "C" fn trampoline_1() {
asm!( asm!(
r#" r#"
# gdb has a hardcoded check that rejects backtraces where frame addresses # gdb has a hardcoded check that rejects backtraces where frame addresses
@ -34,20 +60,21 @@ pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackP
__morestack: __morestack:
.local __morestack .local __morestack
# When a normal function is entered, the return address is pushed onto the stack, # Set up the first part of our DWARF CFI linking stacks together. When
# and the first thing it does is pushing the frame pointer. The init trampoline # we reach this function from unwinding, r2 will be pointing at the bottom
# is not a normal function; on entry the stack pointer is one word above the place # of the parent linked stack. This link is set each time swap() is called.
# where the return address should be, and right under it the return address as # When unwinding the frame corresponding to this function, a DWARF unwinder
# well as the stack pointer are already pre-filled. So, simply move the stack # will use r2+8 as the next call frame address, restore r2 from CFA-4 and
# pointer where it belongs; and add CFI just like in any other function prologue. # restore return address (r9) from CFA-8. This mirrors what the second half
l.addi r1, r1, -8 # of `swap_trampoline` does.
.cfi_def_cfa_offset 8 .cfi_def_cfa r2, 8
.cfi_offset r2, -8 .cfi_offset r2, -4
l.or r2, r1, r0 .cfi_offset r9, -8
.cfi_def_cfa_register r2
# Call f. # This nop is here so that the initial swap doesn't return to the start
l.lwz r9, 8(r1) # of the trampoline, which confuses the unwinder since it will look for
l.jr r9 # frame information in the previous symbol rather than this one. It is
# never actually executed.
l.nop l.nop
.Lend: .Lend:
@ -56,59 +83,98 @@ pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackP
: : : : "volatile") : : : : "volatile")
} }
#[naked]
unsafe extern "C" fn trampoline_2() {
asm!(
r#"
# Set up the second part of our DWARF CFI.
# When unwinding the frame corresponding to this function, a DWARF unwinder
# will restore r2 (and thus CFA of the first trampoline) from the stack slot.
# This stack slot is updated every time swap() is called to point to the bottom
# of the stack of the context switch just switched from.
.cfi_def_cfa r2, 8
.cfi_offset r2, -4
.cfi_offset r9, -8
# Call the provided function.
l.lwz r4, 8(r1)
l.jalr r4
l.nop
"#
: : : : "volatile")
}
unsafe fn push(sp: &mut StackPointer, val: usize) { unsafe fn push(sp: &mut StackPointer, val: usize) {
sp.0 = sp.0.offset(-1); sp.0 = sp.0.offset(-1);
*sp.0 = val *sp.0 = val
} }
// We set up the stack in a somewhat special way so that to the unwinder it
// looks like trampoline_1 has called trampoline_2, which has in turn called
// swap::trampoline.
//
// There are 2 call frames in this setup, each containing the return address
// followed by the r2 value for that frame. This setup supports unwinding
// using DWARF CFI as well as the frame pointer-based unwinding used by tools
// such as perf or dtrace.
let mut sp = StackPointer(stack.base() as *mut usize); let mut sp = StackPointer(stack.base() as *mut usize);
push(&mut sp, f as usize); // function
let rsp = sp; push(&mut sp, f as usize); // Function that trampoline_2 should call
push(&mut sp, trampoline as usize); // trampoline / linked return address
push(&mut sp, 0xdead0bbb); // initial %ebp / linked %ebp // Call frame for trampoline_2. The CFA slot is updated by swap::trampoline
rsp // each time a context switch is performed.
push(&mut sp, 0xdead0cfa); // CFA slot
push(&mut sp, trampoline_1 as usize + 4); // Return after the nop
// Call frame for swap::trampoline. We set up the r2 value to point to the
// parent call frame.
let frame = sp;
push(&mut sp, frame.0 as usize); // Pointer to parent call frame
push(&mut sp, trampoline_2 as usize); // Entry point
// The call frame for swap::trampoline is actually in the red zone and not
// below the stack pointer.
frame
} }
#[inline(always)] #[inline(always)]
pub unsafe fn swap(arg: usize, old_sp: *mut StackPointer, new_sp: StackPointer, pub unsafe fn swap(arg: usize, old_sp: *mut StackPointer, new_sp: StackPointer,
new_stack: &Stack) -> usize { new_stack: &Stack) -> usize {
// Address of the topmost CFA stack slot. // Address of the topmost CFA stack slot.
let new_cfa = (new_stack.base() as *mut usize).offset(-3); let new_cfa = (new_stack.base() as *mut usize).offset(-2);
#[naked] #[naked]
unsafe extern "C" fn trampoline() { unsafe extern "C" fn trampoline() {
asm!( asm!(
r#" r#"
# Remember the frame and instruction pointers in the callee, to link # Save the frame pointer and link register; the unwinder uses them to find
# the stacks together later. # the CFA of the caller, and so they have to have the correct value immediately
l.or r18, r2, r0 # after the call instruction that invoked the trampoline.
l.or r19, r9, r0 l.sw -4(r1), r2
l.sw -8(r1), r9
.cfi_offset r2, -4
.cfi_offset r9, -8
# Save instruction pointer of the old context. # Link the call stacks together by writing the current stack bottom
l.sw -4(r1), r9 # address to the CFA slot in the new stack.
l.addi r7, r1, -8
l.sw 0(r6), r7
# Save frame pointer explicitly; the unwinder uses it to find CFA of # Switch to the new stack for unwinding purposes. The old stack may no
# the caller, and so it has to have the correct value immediately after # longer be valid now that we have modified the link.
# the call instruction that invoked the trampoline. .cfi_def_cfa_register r5
l.sw -8(r1), r2
# Save stack pointer of the old context. # Save stack pointer of the old context.
l.sw 0(r4), r1 l.sw 0(r4), r1
# Load stack pointer of the new context. # Load stack pointer of the new context.
l.or r1, r0, r5 l.or r1, r0, r5
.cfi_def_cfa_register r1
# Load frame and instruction pointers of the new context. # Restore frame pointer and link register of the new context.
l.lwz r2, -8(r1) l.lwz r2, -4(r1)
l.lwz r9, -4(r1) l.lwz r9, -8(r1)
# Put the frame and instruction pointers into the trampoline stack frame, # Return into the new context.
# making it appear to return right after the call instruction that invoked
# this trampoline. This is done after the loads above, since on the very first
# swap, the saved r2/r9 intentionally alias 0(r6)/4(r6).
l.sw 0(r6), r18
l.sw 4(r6), r19
# Return into new context.
l.jr r9 l.jr r9
l.nop l.nop
"# "#
@ -118,8 +184,7 @@ pub unsafe fn swap(arg: usize, old_sp: *mut StackPointer, new_sp: StackPointer,
let ret: usize; let ret: usize;
asm!( asm!(
r#" r#"
# Push instruction pointer of the old context and switch to # Call the trampoline to switch to the new context.
# the new context.
l.jal ${1} l.jal ${1}
l.nop l.nop
"# "#
@ -133,7 +198,7 @@ pub unsafe fn swap(arg: usize, old_sp: *mut StackPointer, new_sp: StackPointer,
"r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
"r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
"r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
"flags", "memory" "cc", "memory"
: "volatile"); : "volatile");
ret ret
} }

160
src/arch/x86.rs
View File

@ -1,6 +1,7 @@
// This file is part of libfringe, a low-level green threading library. // This file is part of libfringe, a low-level green threading library.
// Copyright (c) edef <edef@edef.eu>, // Copyright (c) edef <edef@edef.eu>,
// whitequark <whitequark@whitequark.org> // whitequark <whitequark@whitequark.org>
// Amanieu d'Antras <amanieu@gmail.com>
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be // http://opensource.org/licenses/MIT>, at your option. This file may not be
@ -15,6 +16,31 @@
// * i686 SysV C ABI passes the first argument on the stack. This is // * i686 SysV C ABI passes the first argument on the stack. This is
// unfortunate, because unlike every other architecture we can't reuse // unfortunate, because unlike every other architecture we can't reuse
// `swap` for the initial call, and so we use a trampoline. // `swap` for the initial call, and so we use a trampoline.
//
// To understand the DWARF CFI code in this file, keep in mind these facts:
// * CFI is "call frame information"; a set of instructions to a debugger or
// an unwinder that allow it to simulate returning from functions. This implies
// restoring every register to its pre-call state, as well as the stack pointer.
// * CFA is "call frame address"; the value of stack pointer right before the call
// instruction in the caller. Everything strictly below CFA (and inclusive until
// the next CFA) is the call frame of the callee. This implies that the return
// address is the part of callee's call frame.
// * Logically, DWARF CFI is a table where rows are instruction pointer values and
// columns describe where registers are spilled (mostly using expressions that
// compute a memory location as CFA+n). A .cfi_offset pseudoinstruction changes
// the state of a column for all IP numerically larger than the one it's placed
// after. A .cfi_def_* pseudoinstruction changes the CFA value similarly.
// * Simulating return is as easy as restoring register values from the CFI table
// and then setting stack pointer to CFA.
//
// A high-level overview of the function of the trampolines when unwinding is:
// * The 2nd init trampoline puts a controlled value (written in swap to `new_cfa`)
// into %ebp. This is then used as the CFA for the 1st trampoline.
// * This controlled value points to the bottom of the stack of the parent context,
// which holds the saved %ebp and return address from the call to swap().
// * The 1st init trampoline tells the unwinder to restore %ebp and its return
// address from the stack frame at %ebp (in the parent stack), thus continuing
// unwinding at the swap call site instead of falling off the end of context stack.
use stack::Stack; use stack::Stack;
pub const STACK_ALIGNMENT: usize = 16; pub const STACK_ALIGNMENT: usize = 16;
@ -25,7 +51,7 @@ pub struct StackPointer(*mut usize);
pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackPointer { pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackPointer {
#[cfg(not(target_vendor = "apple"))] #[cfg(not(target_vendor = "apple"))]
#[naked] #[naked]
unsafe extern "C" fn trampoline() { unsafe extern "C" fn trampoline_1() {
asm!( asm!(
r#" r#"
# gdb has a hardcoded check that rejects backtraces where frame addresses # gdb has a hardcoded check that rejects backtraces where frame addresses
@ -36,20 +62,26 @@ pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackP
__morestack: __morestack:
.local __morestack .local __morestack
# When a normal function is entered, the return address is pushed onto the stack, # Set up the first part of our DWARF CFI linking stacks together. When
# and the first thing it does is pushing the frame pointer. The init trampoline # we reach this function from unwinding, %ebp will be pointing at the bottom
# is not a normal function; on entry the stack pointer is one word above the place # of the parent linked stack. This link is set each time swap() is called.
# where the return address should be, and right under it the return address as # When unwinding the frame corresponding to this function, a DWARF unwinder
# well as the stack pointer are already pre-filled. So, simply move the stack # will use %ebp+8 as the next call frame address, restore return address
# pointer where it belongs; and add CFI just like in any other function prologue. # from CFA-4 and restore %ebp from CFA-8. This mirrors what the second half
subl $$8, %esp # of `swap_trampoline` does.
.cfi_def_cfa_offset 8 .cfi_def_cfa ebp, 8
.cfi_offset %ebp, -8 .cfi_offset ebp, -8
movl %esp, %ebp
.cfi_def_cfa_register %ebp # This nop is here so that the initial swap doesn't return to the start
# Call f. # of the trampoline, which confuses the unwinder since it will look for
pushl %eax # frame information in the previous symbol rather than this one. It is
calll *12(%esp) # never actually executed.
nop
# Stack unwinding in some versions of libunwind doesn't seem to like
# 1-byte symbols, so we add a second nop here. This instruction isn't
# executed either, it is only here to pad the symbol size.
nop
.Lend: .Lend:
.size __morestack, .Lend-__morestack .size __morestack, .Lend-__morestack
@ -59,21 +91,36 @@ pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackP
#[cfg(target_vendor = "apple")] #[cfg(target_vendor = "apple")]
#[naked] #[naked]
unsafe extern "C" fn trampoline() { unsafe extern "C" fn trampoline_1() {
asm!( asm!(
r#" r#"
# Identical to the above, except avoids .local/.size that aren't available on Mach-O. # Identical to the above, except avoids .local/.size that aren't available on Mach-O.
__morestack: __morestack:
.private_extern __morestack .private_extern __morestack
.cfi_def_cfa ebp, 8
.cfi_offset ebp, -8
nop
nop
"#
: : : : "volatile")
}
subl $$8, %esp #[naked]
.cfi_def_cfa_offset 8 unsafe extern "C" fn trampoline_2() {
.cfi_offset %ebp, -8 asm!(
movl %esp, %ebp r#"
.cfi_def_cfa_register %ebp # Set up the second part of our DWARF CFI.
# Call f. # When unwinding the frame corresponding to this function, a DWARF unwinder
pushl %eax # will restore %ebp (and thus CFA of the first trampoline) from the stack slot.
calll *12(%esp) # This stack slot is updated every time swap() is called to point to the bottom
# of the stack of the context switch just switched from.
.cfi_def_cfa ebp, 8
.cfi_offset ebp, -8
# Push argument.
pushl %eax
# Call the provided function.
call *12(%esp)
"# "#
: : : : "volatile") : : : : "volatile")
} }
@ -83,10 +130,29 @@ pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackP
*sp.0 = val *sp.0 = val
} }
// We set up the stack in a somewhat special way so that to the unwinder it
// looks like trampoline_1 has called trampoline_2, which has in turn called
// swap::trampoline.
//
// There are 2 call frames in this setup, each containing the return address
// followed by the %ebp value for that frame. This setup supports unwinding
// using DWARF CFI as well as the frame pointer-based unwinding used by tools
// such as perf or dtrace.
let mut sp = StackPointer(stack.base() as *mut usize); let mut sp = StackPointer(stack.base() as *mut usize);
push(&mut sp, f as usize); // function
push(&mut sp, trampoline as usize); // trampoline / linked return address push(&mut sp, f as usize); // Function that trampoline_2 should call
push(&mut sp, 0xdead0bbb); // initial %ebp / linked %ebp
// Call frame for trampoline_2. The CFA slot is updated by swap::trampoline
// each time a context switch is performed.
push(&mut sp, trampoline_1 as usize + 2); // Return after the 2 nops
push(&mut sp, 0xdead0cfa); // CFA slot
// Call frame for swap::trampoline. We set up the %ebp value to point to the
// parent call frame.
let frame = sp;
push(&mut sp, trampoline_2 as usize); // Entry point
push(&mut sp, frame.0 as usize); // Pointer to parent call frame
sp sp
} }
@ -100,38 +166,38 @@ pub unsafe fn swap(arg: usize, old_sp: *mut StackPointer, new_sp: StackPointer,
unsafe extern "C" fn trampoline() { unsafe extern "C" fn trampoline() {
asm!( asm!(
r#" r#"
# Remember the frame and instruction pointers in the callee, to link
# the stacks together later. We put them on stack because x86 doesn't
# have enough registers.
movl %ebp, -8(%edx)
movl (%esp), %ebx
movl %ebx, -12(%edx)
# Save frame pointer explicitly; the unwinder uses it to find CFA of # Save frame pointer explicitly; the unwinder uses it to find CFA of
# the caller, and so it has to have the correct value immediately after # the caller, and so it has to have the correct value immediately after
# the call instruction that invoked the trampoline. # the call instruction that invoked the trampoline.
pushl %ebp pushl %ebp
.cfi_adjust_cfa_offset 4
.cfi_rel_offset ebp, 0
# Link the call stacks together by writing the current stack bottom
# address to the CFA slot in the new stack.
movl %esp, (%edi)
# Switch to the new stack for unwinding purposes. The old stack may no
# longer be valid now that we have modified the link.
.cfi_def_cfa_register edx
# Save stack pointer of the old context. # Save stack pointer of the old context.
movl %esp, (%esi) movl %esp, (%esi)
# Load stack pointer of the new context. # Load stack pointer of the new context.
movl %edx, %esp movl %edx, %esp
.cfi_def_cfa_register esp
# Load frame and instruction pointers of the new context. # Restore frame pointer of the new context.
popl %ebp popl %ebp
popl %ebx .cfi_adjust_cfa_offset -4
.cfi_restore ebp
# Put the frame and instruction pointers into the trampoline stack frame, # Return into the new context. Use `pop` and `jmp` instead of a `ret`
# making it appear to return right after the call instruction that invoked # to avoid return address mispredictions (~8ns per `ret` on Ivy Bridge).
# this trampoline. This is done after the loads above, since on the very first popl %ecx
# swap, the saved %ebp/%ebx intentionally alias 0(%edi)/4(%edi). .cfi_adjust_cfa_offset -4
movl -8(%edx), %esi .cfi_register eip, ecx
movl %esi, 0(%edi) jmpl *%ecx
movl -12(%edx), %esi
movl %esi, 4(%edi)
# Return into new context.
jmpl *%ebx
"# "#
: : : : "volatile") : : : : "volatile")
} }

156
src/arch/x86_64.rs
View File

@ -1,6 +1,7 @@
// This file is part of libfringe, a low-level green threading library. // This file is part of libfringe, a low-level green threading library.
// Copyright (c) edef <edef@edef.eu>, // Copyright (c) edef <edef@edef.eu>,
// whitequark <whitequark@whitequark.org> // whitequark <whitequark@whitequark.org>
// Amanieu d'Antras <amanieu@gmail.com>
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be // http://opensource.org/licenses/MIT>, at your option. This file may not be
@ -19,6 +20,31 @@
// to pass a value while swapping context; this is an arbitrary choice // to pass a value while swapping context; this is an arbitrary choice
// (we clobber all registers and could use any of them) but this allows us // (we clobber all registers and could use any of them) but this allows us
// to reuse the swap function to perform the initial call. // to reuse the swap function to perform the initial call.
//
// To understand the DWARF CFI code in this file, keep in mind these facts:
// * CFI is "call frame information"; a set of instructions to a debugger or
// an unwinder that allow it to simulate returning from functions. This implies
// restoring every register to its pre-call state, as well as the stack pointer.
// * CFA is "call frame address"; the value of stack pointer right before the call
// instruction in the caller. Everything strictly below CFA (and inclusive until
// the next CFA) is the call frame of the callee. This implies that the return
// address is the part of callee's call frame.
// * Logically, DWARF CFI is a table where rows are instruction pointer values and
// columns describe where registers are spilled (mostly using expressions that
// compute a memory location as CFA+n). A .cfi_offset pseudoinstruction changes
// the state of a column for all IP numerically larger than the one it's placed
// after. A .cfi_def_* pseudoinstruction changes the CFA value similarly.
// * Simulating return is as easy as restoring register values from the CFI table
// and then setting stack pointer to CFA.
//
// A high-level overview of the function of the trampolines when unwinding is:
// * The 2nd init trampoline puts a controlled value (written in swap to `new_cfa`)
// into %rbp. This is then used as the CFA for the 1st trampoline.
// * This controlled value points to the bottom of the stack of the parent context,
// which holds the saved %rbp and return address from the call to swap().
// * The 1st init trampoline tells the unwinder to restore %rbp and its return
// address from the stack frame at %rbp (in the parent stack), thus continuing
// unwinding at the swap call site instead of falling off the end of context stack.
use stack::Stack; use stack::Stack;
pub const STACK_ALIGNMENT: usize = 16; pub const STACK_ALIGNMENT: usize = 16;
@ -29,7 +55,7 @@ pub struct StackPointer(*mut usize);
pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackPointer { pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackPointer {
#[cfg(not(target_vendor = "apple"))] #[cfg(not(target_vendor = "apple"))]
#[naked] #[naked]
unsafe extern "C" fn trampoline() { unsafe extern "C" fn trampoline_1() {
asm!( asm!(
r#" r#"
# gdb has a hardcoded check that rejects backtraces where frame addresses # gdb has a hardcoded check that rejects backtraces where frame addresses
@ -40,19 +66,26 @@ pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackP
__morestack: __morestack:
.local __morestack .local __morestack
# When a normal function is entered, the return address is pushed onto the stack, # Set up the first part of our DWARF CFI linking stacks together. When
# and the first thing it does is pushing the frame pointer. The init trampoline # we reach this function from unwinding, %rbp will be pointing at the bottom
# is not a normal function; on entry the stack pointer is one word above the place # of the parent linked stack. This link is set each time swap() is called.
# where the return address should be, and right under it the return address as # When unwinding the frame corresponding to this function, a DWARF unwinder
# well as the stack pointer are already pre-filled. So, simply move the stack # will use %rbp+16 as the next call frame address, restore return address
# pointer where it belongs; and add CFI just like in any other function prologue. # from CFA-8 and restore %rbp from CFA-16. This mirrors what the second half
subq $$16, %rsp # of `swap_trampoline` does.
.cfi_def_cfa_offset 16 .cfi_def_cfa rbp, 16
.cfi_offset %rbp, -16 .cfi_offset rbp, -16
movq %rsp, %rbp
.cfi_def_cfa_register %rbp # This nop is here so that the initial swap doesn't return to the start
# Call f. # of the trampoline, which confuses the unwinder since it will look for
callq *16(%rsp) # frame information in the previous symbol rather than this one. It is
# never actually executed.
nop
# Stack unwinding in some versions of libunwind doesn't seem to like
# 1-byte symbols, so we add a second nop here. This instruction isn't
# executed either, it is only here to pad the symbol size.
nop
.Lend: .Lend:
.size __morestack, .Lend-__morestack .size __morestack, .Lend-__morestack
@ -62,32 +95,67 @@ pub unsafe fn init(stack: &Stack, f: unsafe extern "C" fn(usize) -> !) -> StackP
#[cfg(target_vendor = "apple")] #[cfg(target_vendor = "apple")]
#[naked] #[naked]
unsafe extern "C" fn trampoline() { unsafe extern "C" fn trampoline_1() {
asm!( asm!(
r#" r#"
# Identical to the above, except avoids .local/.size that aren't available on Mach-O. # Identical to the above, except avoids .local/.size that aren't available on Mach-O.
__morestack: __morestack:
.private_extern __morestack .private_extern __morestack
.cfi_def_cfa rbp, 16
subq $$16, %rsp .cfi_offset rbp, -16
.cfi_def_cfa_offset 16 nop
.cfi_offset %rbp, -16 nop
movq %rsp, %rbp
.cfi_def_cfa_register %rbp
callq *16(%rsp)
"# "#
: : : : "volatile") : : : : "volatile")
} }
#[naked]
unsafe extern "C" fn trampoline_2() {
asm!(
r#"
# Set up the second part of our DWARF CFI.
# When unwinding the frame corresponding to this function, a DWARF unwinder
# will restore %rbp (and thus CFA of the first trampoline) from the stack slot.
# This stack slot is updated every time swap() is called to point to the bottom
# of the stack of the context switch just switched from.
.cfi_def_cfa rbp, 16
.cfi_offset rbp, -16
# Call the provided function.
call *16(%rsp)
"#
: : : : "volatile")
}
unsafe fn push(sp: &mut StackPointer, val: usize) { unsafe fn push(sp: &mut StackPointer, val: usize) {
sp.0 = sp.0.offset(-1); sp.0 = sp.0.offset(-1);
*sp.0 = val *sp.0 = val
} }
// We set up the stack in a somewhat special way so that to the unwinder it
// looks like trampoline_1 has called trampoline_2, which has in turn called
// swap::trampoline.
//
// There are 2 call frames in this setup, each containing the return address
// followed by the %rbp value for that frame. This setup supports unwinding
// using DWARF CFI as well as the frame pointer-based unwinding used by tools
// such as perf or dtrace.
let mut sp = StackPointer(stack.base() as *mut usize); let mut sp = StackPointer(stack.base() as *mut usize);
push(&mut sp, 0 as usize); // alignment
push(&mut sp, f as usize); // function push(&mut sp, 0 as usize); // Padding to ensure the stack is properly aligned
push(&mut sp, trampoline as usize); // trampoline / linked return address push(&mut sp, f as usize); // Function that trampoline_2 should call
push(&mut sp, 0xdeaddeaddead0bbb); // initial %rbp / linked %rbp
// Call frame for trampoline_2. The CFA slot is updated by swap::trampoline
// each time a context switch is performed.
push(&mut sp, trampoline_1 as usize + 2); // Return after the 2 nops
push(&mut sp, 0xdeaddeaddead0cfa); // CFA slot
// Call frame for swap::trampoline. We set up the %rbp value to point to the
// parent call frame.
let frame = sp;
push(&mut sp, trampoline_2 as usize); // Entry point
push(&mut sp, frame.0 as usize); // Pointer to parent call frame
sp sp
} }
@ -101,34 +169,38 @@ pub unsafe fn swap(arg: usize, old_sp: *mut StackPointer, new_sp: StackPointer,
unsafe extern "C" fn trampoline() { unsafe extern "C" fn trampoline() {
asm!( asm!(
r#" r#"
# Remember the frame and instruction pointers in the callee, to link
# the stacks together later.
movq %rbp, %r8
movq (%rsp), %r9
# Save frame pointer explicitly; the unwinder uses it to find CFA of # Save frame pointer explicitly; the unwinder uses it to find CFA of
# the caller, and so it has to have the correct value immediately after # the caller, and so it has to have the correct value immediately after
# the call instruction that invoked the trampoline. # the call instruction that invoked the trampoline.
pushq %rbp pushq %rbp
.cfi_adjust_cfa_offset 8
.cfi_rel_offset rbp, 0
# Link the call stacks together by writing the current stack bottom
# address to the CFA slot in the new stack.
movq %rsp, (%rcx)
# Switch to the new stack for unwinding purposes. The old stack may no
# longer be valid now that we have modified the link.
.cfi_def_cfa_register rdx
# Save stack pointer of the old context. # Save stack pointer of the old context.
movq %rsp, (%rsi) movq %rsp, (%rsi)
# Load stack pointer of the new context. # Load stack pointer of the new context.
movq %rdx, %rsp movq %rdx, %rsp
.cfi_def_cfa_register rsp
# Load frame and instruction pointers of the new context. # Restore frame pointer of the new context.
popq %rbp popq %rbp
popq %rbx .cfi_adjust_cfa_offset -8
.cfi_restore rbp
# Put the frame and instruction pointers into the trampoline stack frame, # Return into the new context. Use `pop` and `jmp` instead of a `ret`
# making it appear to return right after the call instruction that invoked # to avoid return address mispredictions (~8ns per `ret` on Ivy Bridge).
# this trampoline. This is done after the loads above, since on the very first popq %rax
# swap, the saved %rbp/%rbx intentionally alias 0(%rcx)/8(%rcx). .cfi_adjust_cfa_offset -8
movq %r8, 0(%rcx) .cfi_register rip, rax
movq %r9, 8(%rcx) jmpq *%rax
# Return into new context.
jmpq *%rbx
"# "#
: : : : "volatile") : : : : "volatile")
} }