From 87bd2072e8698aa9777739cf3993fa0efd519dd4 Mon Sep 17 00:00:00 2001 From: Robert Jordens Date: Fri, 2 Dec 2016 14:33:26 +0100 Subject: [PATCH] sawg: round to int64 --- artiq/coredevice/sawg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/artiq/coredevice/sawg.py b/artiq/coredevice/sawg.py index 91062e6a6..2b4feff98 100644 --- a/artiq/coredevice/sawg.py +++ b/artiq/coredevice/sawg.py @@ -12,9 +12,10 @@ class Spline: self.core = core_device self.channel = channel self.width = width - self.scale = (1 << width) * scale + self.scale = float((int64(1) << width) * scale) self.time_width = time_width - self.time_scale = (1 << time_width) * core_device.coarse_ref_period + self.time_scale = float((1 << time_width) * + core_device.coarse_ref_period) @portable(flags={"fast-math"}) def to_mu(self, value: TFloat) -> TInt32: