runtime/eh_artiq: use forced unwind

This patches ports the LLVM libunwind newly added forced unwinding
function. This enables us to run forced unwinding to obtain correct
backtrace when uncaught exceptions occur.

This patch also changes the exception handling scheme from the standard
two-phase unwinding to single phase using forced unwinding. This brings
some performance improvement and prepared for later nested exception
support. For nested exceptions, we will have to record the backtrace
regardless if the exception is an uncaught exception, as there can be
another exception being thrown while executing the finally block for
caught exceptions, and we will lose the backtrace if we don't store it
earlier before running the cleanup pads.

src/llvm_libunwind/include/unwind.h

@@ -107,9 +107,12 @@ struct _Unwind_Control_Block {
 } __attribute__((__aligned__(8)));
 
 typedef _Unwind_Reason_Code (*_Unwind_Stop_Fn)
-      (_Unwind_State state,
+    (int version,
+     _Unwind_Action actions,
+     uint64_t exceptionClass,
      _Unwind_Exception* exceptionObject,
-       struct _Unwind_Context* context);
+     struct _Unwind_Context* context,
+     void* stop_parameter);
 
 typedef _Unwind_Reason_Code (*__personality_routine)
       (_Unwind_State state,

src/llvm_libunwind/src/Unwind-EHABI.cpp

@@ -95,9 +95,11 @@ _Unwind_Reason_Code ProcessDescriptors(
       case Descriptor::LU32:
         descriptor = getNextWord(descriptor, &length);
         descriptor = getNextWord(descriptor, &offset);
+        break;
       case Descriptor::LU16:
         descriptor = getNextNibble(descriptor, &length);
         descriptor = getNextNibble(descriptor, &offset);
+        break;
       default:
         assert(false);
         return _URC_FAILURE;
@@ -183,8 +185,14 @@ static _Unwind_Reason_Code unwindOneFrame(_Unwind_State state,
   if (result != _URC_CONTINUE_UNWIND)
     return result;
 
-  if (__unw_step(reinterpret_cast<unw_cursor_t *>(context)) != UNW_STEP_SUCCESS)
+  switch (__unw_step(reinterpret_cast<unw_cursor_t *>(context))) {
+  case UNW_STEP_SUCCESS:
+    return _URC_CONTINUE_UNWIND;
+  case UNW_STEP_END:
+    return _URC_END_OF_STACK;
+  default:
     return _URC_FAILURE;
+  }
   return _URC_CONTINUE_UNWIND;
 }
 
@@ -677,6 +685,128 @@ static _Unwind_Reason_Code unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor
   return _URC_FATAL_PHASE2_ERROR;
 }
 
+static _Unwind_Reason_Code
+unwind_phase2_forced(unw_context_t *uc, unw_cursor_t *cursor,
+                     _Unwind_Exception *exception_object, _Unwind_Stop_Fn stop,
+                     void *stop_parameter) {
+  // See comment at the start of unwind_phase1 regarding VRS integrity.
+  __unw_init_local(cursor, uc);
+
+  _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p)",
+                             static_cast<void *>(exception_object));
+
+  // Walk each frame until we reach where search phase said to stop.
+  bool end_of_stack = false;
+  // TODO: why can't libunwind handle end of stack properly?
+  // We should fix this kind of hack.
+  unw_word_t forced_phase2_prev_sp = 0x0;
+  while (!end_of_stack) {
+    // Get info about this frame.
+    unw_word_t sp;
+    unw_proc_info_t frameInfo;
+    __unw_get_reg(cursor, UNW_REG_SP, &sp);
+    if (sp == forced_phase2_prev_sp) {
+        break;
+    }
+    forced_phase2_prev_sp = sp;
+    if (__unw_get_proc_info(cursor, &frameInfo) != UNW_ESUCCESS) {
+      _LIBUNWIND_TRACE_UNWINDING(
+          "unwind_phase2_forced(ex_ojb=%p): __unw_get_proc_info "
+          "failed => _URC_FATAL_PHASE2_ERROR",
+          static_cast<void *>(exception_object));
+      return _URC_FATAL_PHASE2_ERROR;
+    }
+
+    // When tracing, print state information.
+    if (_LIBUNWIND_TRACING_UNWINDING) {
+      char functionBuf[512];
+      const char *functionName = functionBuf;
+      unw_word_t offset;
+      if ((__unw_get_proc_name(cursor, functionBuf, sizeof(functionBuf),
+                               &offset) != UNW_ESUCCESS) ||
+          (frameInfo.start_ip + offset > frameInfo.end_ip))
+        functionName = ".anonymous.";
+      _LIBUNWIND_TRACE_UNWINDING(
+          "unwind_phase2_forced(ex_ojb=%p): start_ip=0x%" PRIxPTR ", func=%s, sp=0x%" PRIxPTR ", "
+          "lsda=0x%" PRIxPTR ", personality=0x%" PRIxPTR "",
+          static_cast<void *>(exception_object), frameInfo.start_ip,
+          functionName, sp, frameInfo.lsda,
+          frameInfo.handler);
+    }
+
+    _Unwind_Action action =
+        (_Unwind_Action)(_UA_FORCE_UNWIND | _UA_CLEANUP_PHASE);
+    _Unwind_Reason_Code stopResult =
+        (*stop)(1, action, exception_object->exception_class, exception_object,
+                (_Unwind_Context *)(cursor), stop_parameter);
+
+    _LIBUNWIND_TRACE_UNWINDING(
+        "unwind_phase2_forced(ex_ojb=%p): stop function returned %d",
+        (void *)exception_object, stopResult);
+    if (stopResult != _URC_NO_REASON) {
+      _LIBUNWIND_TRACE_UNWINDING(
+          "unwind_phase2_forced(ex_ojb=%p): stopped by stop function",
+          (void *)exception_object);
+      return _URC_FATAL_PHASE2_ERROR;
+    }
+
+    // If there is a personality routine, tell it we are unwinding.
+    if (frameInfo.handler != 0) {
+      __personality_routine p =
+          (__personality_routine)(long)(frameInfo.handler);
+      struct _Unwind_Context *context = (struct _Unwind_Context *)(cursor);
+      // EHABI #7.2
+      exception_object->pr_cache.fnstart = frameInfo.start_ip;
+      exception_object->pr_cache.ehtp =
+          (_Unwind_EHT_Header *)frameInfo.unwind_info;
+      exception_object->pr_cache.additional = frameInfo.flags;
+      _Unwind_Reason_Code personalityResult =
+          (*p)(_US_FORCE_UNWIND | _US_UNWIND_FRAME_STARTING, exception_object,
+              context);
+      switch (personalityResult) {
+      case _URC_CONTINUE_UNWIND:
+        // Continue unwinding
+        _LIBUNWIND_TRACE_UNWINDING(
+            "unwind_phase2_forced(ex_ojb=%p): _URC_CONTINUE_UNWIND",
+            static_cast<void *>(exception_object));
+        break;
+      case _URC_INSTALL_CONTEXT:
+        _LIBUNWIND_TRACE_UNWINDING(
+            "unwind_phase2_forced(ex_ojb=%p): _URC_INSTALL_CONTEXT",
+            static_cast<void *>(exception_object));
+        {
+          // EHABI #7.4.1 says we need to preserve pc for when _Unwind_Resume
+          // is called back, to find this same frame.
+          unw_word_t pc;
+          __unw_get_reg(cursor, UNW_REG_IP, &pc);
+          exception_object->unwinder_cache.reserved2 = (uint32_t)pc;
+        }
+        // We may get control back if landing pad calls _Unwind_Resume().
+        __unw_resume(cursor);
+        break;
+      case _URC_END_OF_STACK:
+        end_of_stack = true;
+        break;
+      default:
+        // Personality routine returned an unknown result code.
+        _LIBUNWIND_DEBUG_LOG("personality function returned unknown result %d",
+                      personalityResult);
+        return _URC_FATAL_PHASE2_ERROR;
+      }
+    }
+  }
+  _LIBUNWIND_TRACE_UNWINDING("unwind_phase2_forced(ex_ojb=%p): calling stop "
+                             "function with _UA_END_OF_STACK",
+                             (void *)exception_object);
+  _Unwind_Action lastAction =
+      (_Unwind_Action)(_UA_FORCE_UNWIND | _UA_CLEANUP_PHASE | _UA_END_OF_STACK);
+  (*stop)(1, lastAction, exception_object->exception_class, exception_object,
+          (struct _Unwind_Context *)(cursor), stop_parameter);
+  return _URC_FATAL_PHASE2_ERROR;
+}
+
+
+
 /// Called by __cxa_throw.  Only returns if there is a fatal error.
 _LIBUNWIND_EXPORT _Unwind_Reason_Code
 _Unwind_RaiseException(_Unwind_Exception *exception_object) {
@@ -724,15 +854,36 @@ _Unwind_Resume(_Unwind_Exception *exception_object) {
   unw_cursor_t cursor;
   __unw_getcontext(&uc);
 
-  // _Unwind_RaiseException on EHABI will always set the reserved1 field to 0,
+  if (exception_object->unwinder_cache.reserved1)
-  // which is in the same position as private_1 below.
+    unwind_phase2_forced(
-  // TODO(ajwong): Who wronte the above? Why is it true?
+        &uc, &cursor, exception_object,
+        (_Unwind_Stop_Fn)exception_object->unwinder_cache.reserved1,
+        (void *)exception_object->unwinder_cache.reserved3);
+  else
     unwind_phase2(&uc, &cursor, exception_object, true);
 
   // Clients assume _Unwind_Resume() does not return, so all we can do is abort.
   _LIBUNWIND_ABORT("_Unwind_Resume() can't return");
 }
 
+_LIBUNWIND_EXPORT _Unwind_Reason_Code
+_Unwind_ForcedUnwind(_Unwind_Exception *exception_object, _Unwind_Stop_Fn stop,
+                     void *stop_parameter) {
+  _LIBUNWIND_TRACE_API("_Unwind_ForcedUnwind(ex_obj=%p, stop=%p)",
+                       (void *)exception_object, (void *)(uintptr_t)stop);
+  unw_context_t uc;
+  unw_cursor_t cursor;
+  __unw_getcontext(&uc);
+
+  // Mark that this is a forced unwind, so _Unwind_Resume() can do
+  // the right thing.
+  exception_object->unwinder_cache.reserved1 = (uintptr_t)stop;
+  exception_object->unwinder_cache.reserved3 = (uintptr_t)stop_parameter;
+
+  return unwind_phase2_forced(&uc, &cursor, exception_object, stop,
+                              stop_parameter);
+}
+
 /// Called by personality handler during phase 2 to get LSDA for current frame.
 _LIBUNWIND_EXPORT uintptr_t
 _Unwind_GetLanguageSpecificData(struct _Unwind_Context *context) {
@@ -1002,9 +1153,14 @@ extern "C" _LIBUNWIND_EXPORT _Unwind_Reason_Code
 __gnu_unwind_frame(_Unwind_Exception *exception_object,
                    struct _Unwind_Context *context) {
   unw_cursor_t *cursor = (unw_cursor_t *)context;
-  if (__unw_step(cursor) != UNW_STEP_SUCCESS)
+  switch (__unw_step(cursor)) {
-    return _URC_FAILURE;
+  case UNW_STEP_SUCCESS:
     return _URC_OK;
+  case UNW_STEP_END:
+    return _URC_END_OF_STACK;
+  default:
+    return _URC_FAILURE;
+  }
 }
 
 #endif  // defined(_LIBUNWIND_ARM_EHABI)

src/runtime/src/eh_artiq.rs

@@ -15,7 +15,7 @@
 use core::mem;
 use cslice::CSlice;
 use unwind as uw;
-use libc::{c_int, uintptr_t};
+use libc::{c_int, c_void, uintptr_t};
 use log::trace;
 
 use dwarf::eh::{self, EHAction, EHContext};
@@ -59,11 +59,25 @@ const MAX_BACKTRACE_SIZE: usize = 128;
 struct ExceptionInfo {
     uw_exception:   uw::_Unwind_Exception,
     exception:      Option<Exception<'static>>,
-    handled:        bool,
     backtrace:      [usize; MAX_BACKTRACE_SIZE],
     backtrace_size: usize
 }
 
+type _Unwind_Stop_Fn = extern "C" fn(version: c_int,
+                                     actions: i32,
+                                     exception_class: uw::_Unwind_Exception_Class,
+                                     exception_object: *mut uw::_Unwind_Exception,
+                                     context: *mut uw::_Unwind_Context,
+                                     stop_parameter: *mut c_void)
+                                    -> uw::_Unwind_Reason_Code;
+
+extern {
+    // not defined in EHABI, but LLVM added it and is useful to us
+    fn _Unwind_ForcedUnwind(exception: *mut uw::_Unwind_Exception,
+                            stop_fn: _Unwind_Stop_Fn,
+                            stop_parameter: *mut c_void) -> uw::_Unwind_Reason_Code;
+}
+
 unsafe fn find_eh_action(
     context: *mut uw::_Unwind_Context,
     foreign_exception: bool,
@@ -84,29 +98,11 @@ unsafe fn find_eh_action(
     eh::find_eh_action(lsda, &eh_context, foreign_exception, name, len)
 }
 
-pub unsafe fn artiq_personality(state: uw::_Unwind_State,
+pub unsafe fn artiq_personality(_state: uw::_Unwind_State,
                                 exception_object: *mut uw::_Unwind_Exception,
                                 context: *mut uw::_Unwind_Context)
                                 -> uw::_Unwind_Reason_Code {
-    let state = state as c_int;
+    // we will only do phase 2 forced unwinding now
-    let action = state & uw::_US_ACTION_MASK as c_int;
-    let search_phase = if action == uw::_US_VIRTUAL_UNWIND_FRAME as c_int {
-        // Backtraces on ARM will call the personality routine with
-        // state == _US_VIRTUAL_UNWIND_FRAME | _US_FORCE_UNWIND. In those cases
-        // we want to continue unwinding the stack, otherwise all our backtraces
-        // would end at __rust_try
-        if state & uw::_US_FORCE_UNWIND as c_int != 0 {
-            return continue_unwind(exception_object, context);
-        }
-        true
-    } else if action == uw::_US_UNWIND_FRAME_STARTING as c_int {
-        false
-    } else if action == uw::_US_UNWIND_FRAME_RESUME as c_int {
-        return continue_unwind(exception_object, context);
-    } else {
-        return uw::_URC_FAILURE;
-    };
-
     // The DWARF unwinder assumes that _Unwind_Context holds things like the function
     // and LSDA pointers, however ARM EHABI places them into the exception object.
     // To preserve signatures of functions like _Unwind_GetLanguageSpecificData(), which
@@ -122,6 +118,7 @@ pub unsafe fn artiq_personality(state: uw::_Unwind_State,
 
     let exception_class = (*exception_object).exception_class;
     let foreign_exception = exception_class != EXCEPTION_CLASS;
+    assert!(!foreign_exception, "we do not expect foreign exceptions");
     let exception_info = &mut *(exception_object as *mut ExceptionInfo);
 
     let (name_ptr, len) = if foreign_exception || exception_info.exception.is_none() {
@@ -135,27 +132,6 @@ pub unsafe fn artiq_personality(state: uw::_Unwind_State,
         Err(_) => return uw::_URC_FAILURE,
     };
     let exception = &exception_info.exception.unwrap();
-    if search_phase {
-        match eh_action {
-            EHAction::None => return continue_unwind(exception_object, context),
-            // Actually, cleanup should not return handler found, this is to workaround
-            // the issue of terminating directly when no catch cause is found while
-            // having some cleanup routines defined by finally.
-            // The best way to handle this is to force unwind the stack in the raise
-            // function when end of stack is reached, and call terminate at the end of
-            // the unwind. Unfortunately, there is no forced unwind function defined
-            // for EHABI, and I have no idea how to implement that, so this is a hack.
-            EHAction::Cleanup(_) => return uw::_URC_HANDLER_FOUND,
-            EHAction::Catch(_) => {
-                // EHABI requires the personality routine to update the
-                // SP value in the barrier cache of the exception object.
-                (*exception_object).private[5] =
-                    uw::_Unwind_GetGR(context, uw::UNWIND_SP_REG);
-                return uw::_URC_HANDLER_FOUND;
-            }
-            EHAction::Terminate => return uw::_URC_FAILURE,
-        }
-    } else {
     match eh_action {
         EHAction::None => return continue_unwind(exception_object, context),
         EHAction::Cleanup(lpad) |
@@ -168,17 +144,17 @@ pub unsafe fn artiq_personality(state: uw::_Unwind_State,
         }
         EHAction::Terminate => return uw::_URC_FAILURE,
     }
-    }
 
     // On ARM EHABI the personality routine is responsible for actually
     // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1).
     unsafe fn continue_unwind(exception_object: *mut uw::_Unwind_Exception,
                               context: *mut uw::_Unwind_Context)
                               -> uw::_Unwind_Reason_Code {
-        if __gnu_unwind_frame(exception_object, context) == uw::_URC_NO_REASON {
+        let reason = __gnu_unwind_frame(exception_object, context);
+        if reason == uw::_URC_NO_REASON {
             uw::_URC_CONTINUE_UNWIND
         } else {
-            uw::_URC_FAILURE
+            reason
         }
     }
     // defined in libgcc
@@ -205,30 +181,47 @@ static mut INFLIGHT: ExceptionInfo = ExceptionInfo {
         private:           [0; uw::unwinder_private_data_size],
     },
     exception:      None,
-    handled:        true,
     backtrace:      [0; MAX_BACKTRACE_SIZE],
     backtrace_size: 0
 };
 
 pub unsafe extern fn raise(exception: *const Exception) -> ! {
-    trace!("Trying to raise exception");
     // FIXME: unsound transmute
     // This would cause stack memory corruption.
-    INFLIGHT.exception = Some(mem::transmute::<Exception, Exception<'static>>(*exception));
+    trace!("raising exception");
-    INFLIGHT.handled   = false;
-
-    let result = uw::_Unwind_RaiseException(&mut INFLIGHT.uw_exception);
-    assert!(result == uw::_URC_FAILURE || result == uw::_URC_END_OF_STACK);
-
     INFLIGHT.backtrace_size = 0;
-    // read backtrace
+    INFLIGHT.exception = Some(mem::transmute::<Exception, Exception<'static>>(*exception));
-    let _ = uw::backtrace(|ip| {
+
-        if INFLIGHT.backtrace_size < MAX_BACKTRACE_SIZE {
+    let _result = _Unwind_ForcedUnwind(&mut INFLIGHT.uw_exception,
-            INFLIGHT.backtrace[INFLIGHT.backtrace_size] = ip;
+                                   uncaught_exception, core::ptr::null_mut());
-            INFLIGHT.backtrace_size += 1;
+    unreachable!()
+}
+
+extern fn uncaught_exception(_version: c_int,
+                             actions: i32,
+                             _uw_exception_class: uw::_Unwind_Exception_Class,
+                             uw_exception: *mut uw::_Unwind_Exception,
+                             context: *mut uw::_Unwind_Context,
+                             _stop_parameter: *mut c_void)
+                            -> uw::_Unwind_Reason_Code {
+    unsafe {
+        trace!("uncaught exception");
+        let exception_info = &mut *(uw_exception as *mut ExceptionInfo);
+
+        if exception_info.backtrace_size < exception_info.backtrace.len() {
+            let ip = uw::_Unwind_GetIP(context);
+            trace!("SP: {:X}, backtrace_size: {}", uw::_Unwind_GetGR(context, uw::UNWIND_SP_REG), exception_info.backtrace_size);
+            exception_info.backtrace[exception_info.backtrace_size] = ip;
+            exception_info.backtrace_size += 1;
+        }
+
+        if actions as u32 & uw::_US_END_OF_STACK as u32 != 0 {
+            crate::kernel::core1::terminate(exception_info.exception.as_ref().unwrap(),
+                        exception_info.backtrace[..exception_info.backtrace_size].as_mut())
+        } else {
+            uw::_URC_NO_REASON
+        }
     }
-    });
-    crate::kernel::core1::terminate(INFLIGHT.exception.as_ref().unwrap(), INFLIGHT.backtrace[..INFLIGHT.backtrace_size].as_mut());
 }
 
 pub unsafe extern fn reraise() -> ! {
@@ -237,8 +230,15 @@ pub unsafe extern fn reraise() -> ! {
     // Reraise is basically cxa_rethrow, which calls _Unwind_Resume_or_Rethrow,
     // which for EHABI would always call _Unwind_RaiseException.
     match INFLIGHT.exception {
-        Some(ref exception) => raise(exception),
+        Some(ex) => {
-        None => raise(&Exception {
+            // we cannot call raise directly as that would corrupt the backtrace
+            INFLIGHT.exception = Some(mem::transmute::<Exception, Exception<'static>>(ex));
+            let _result = _Unwind_ForcedUnwind(&mut INFLIGHT.uw_exception,
+                                           uncaught_exception, core::ptr::null_mut());
+            unreachable!()
+        },
+        None => {
+            raise(&Exception {
                 name:     "0:artiq.coredevice.exceptions.RuntimeError".as_c_slice(),
                 file:     file!().as_c_slice(),
                 line:     line!(),
@@ -250,6 +250,7 @@ pub unsafe extern fn reraise() -> ! {
             })
         }
     }
+}
 
 #[macro_export]
 macro_rules! artiq_raise {
@@ -266,7 +267,9 @@ macro_rules! artiq_raise {
             param:    [$param0, $param1, $param2]
         };
         #[allow(unused_unsafe)]
-        unsafe { $crate::eh_artiq::raise(&exn) }
+        unsafe {
+            $crate::eh_artiq::raise(&exn)
+        }
     });
     ($name:expr, $message:expr) => ({
         artiq_raise!($name, $message, 0, 0, 0)