diff --git a/build.rs b/build.rs
index 171622c..e66b3ab 100644
--- a/build.rs
+++ b/build.rs
@@ -4132,7 +4132,6 @@ mod c {
             if target_arch == "x86_64" {
                 sources.extend(
                     &[
-                        "x86_64/floatdidf.c",
                         "x86_64/floatdisf.c",
                         "x86_64/floatdixf.c",
                     ],
@@ -4148,7 +4147,6 @@ mod c {
                     &[
                         "x86_64/chkstk.S",
                         "x86_64/chkstk2.S",
-                        "x86_64/floatdidf.c",
                         "x86_64/floatdisf.c",
                         "x86_64/floatdixf.c",
                         "x86_64/floatundidf.S",
diff --git a/src/float/conv.rs b/src/float/conv.rs
index ca95b11..ca05c28 100644
--- a/src/float/conv.rs
+++ b/src/float/conv.rs
@@ -78,12 +78,15 @@ intrinsics! {
         int_to_float!(i, i32, f64)
     }
 
-    #[use_c_shim_if(any(
-        all(not(target_env = "msvc"), target_arch = "x86", target_arch = "x86_64"),
-        all(target_env = "msvc", target_arch = "x86_64"),
-    ))]
+    #[use_c_shim_if(all(target_arch = "x86", not(target_env = "msvc")))]
     pub extern "C" fn __floatdidf(i: i64) -> f64 {
-        int_to_float!(i, i64, f64)
+        // On x86_64 LLVM will use native instructions for this conversion, we
+        // can just do it directly
+        if cfg!(target_arch = "x86_64") {
+            i as f64
+        } else {
+            int_to_float!(i, i64, f64)
+        }
     }
 
     #[unadjusted_on_win64]