forked from M-Labs/artiq
1
0
Fork 0

test: improved test_performance

1. Added tests for small payload.
2. Added statistics.
This commit is contained in:
pca006132 2020-08-27 13:06:09 +08:00 committed by Sébastien Bourdeauducq
parent 7cf974a6a7
commit 69f0699ebd
1 changed files with 220 additions and 90 deletions

View File

@ -6,138 +6,268 @@ import numpy
from artiq.experiment import * from artiq.experiment import *
from artiq.test.hardware_testbench import ExperimentCase from artiq.test.hardware_testbench import ExperimentCase
# large: 1MB payload
# small: 1KB payload
bytes_large = b"\x00" * (1 << 20)
bytes_small = b"\x00" * (1 << 10)
list_large = [123] * (1 << 18)
list_small = [123] * (1 << 8)
array_large = numpy.array(list_large, numpy.int32)
array_small = numpy.array(list_small, numpy.int32)
byte_list_large = [True] * (1 << 20)
byte_list_small = [True] * (1 << 10)
received_bytes = 0
time_start = 0
time_end = 0
class _Transfer(EnvExperiment): class _Transfer(EnvExperiment):
def build(self): def build(self):
self.setattr_device("core") self.setattr_device("core")
self.data = b"\x00"*(10**6) self.count = 10
self.h2d = [0.0] * self.count
self.d2h = [0.0] * self.count
@rpc @rpc
def source(self) -> TBytes: def get_bytes(self, large: TBool) -> TBytes:
return self.data if large:
return bytes_large
else:
return bytes_small
@rpc @rpc
def source_byte_list(self) -> TList(TBool): def get_list(self, large: TBool) -> TList(TInt32):
return [True] * (1 << 15) if large:
return list_large
else:
return list_small
@rpc @rpc
def source_list(self) -> TList(TInt32): def get_byte_list(self, large: TBool) -> TList(TBool):
return [123] * (1 << 15) if large:
return byte_list_large
else:
return byte_list_small
@rpc @rpc
def source_array(self) -> TArray(TInt32): def get_array(self, large: TBool) -> TArray(TInt32):
return numpy.array([0] * (1 << 15), numpy.int32) if large:
return array_large
else:
return array_small
@rpc
def get_string_list(self) -> TList(TStr):
return string_list
@rpc @rpc
def sink(self, data): def sink(self, data):
pass pass
@rpc @rpc(flags={"async"})
def sink_list(self, data): def sink_async(self, data):
pass global received_bytes, time_start, time_end
if received_bytes == 0:
time_start = time.time()
received_bytes += len(data)
if received_bytes == (1024 ** 2)*128:
time_end = time.time()
@rpc @rpc
def sink_array(self, data): def get_async_throughput(self) -> TFloat:
pass return 128.0 / (time_end - time_start)
@kernel @kernel
def host_to_device(self): def test_bytes(self, large):
def inner():
t0 = self.core.get_rtio_counter_mu() t0 = self.core.get_rtio_counter_mu()
data = self.source() data = self.get_bytes(large)
t1 = self.core.get_rtio_counter_mu() t1 = self.core.get_rtio_counter_mu()
return len(data)/self.core.mu_to_seconds(t1-t0) self.sink(data)
t2 = self.core.get_rtio_counter_mu()
self.h2d[i] = self.core.mu_to_seconds(t1 - t0)
self.d2h[i] = self.core.mu_to_seconds(t2 - t1)
for i in range(self.count):
inner()
return (self.h2d, self.d2h)
@kernel @kernel
def host_to_device_list(self): def test_byte_list(self, large):
def inner():
t0 = self.core.get_rtio_counter_mu() t0 = self.core.get_rtio_counter_mu()
data = self.source_list() data = self.get_byte_list(large)
t1 = self.core.get_rtio_counter_mu() t1 = self.core.get_rtio_counter_mu()
return 4 * len(data)/self.core.mu_to_seconds(t1-t0) self.sink(data)
t2 = self.core.get_rtio_counter_mu()
self.h2d[i] = self.core.mu_to_seconds(t1 - t0)
self.d2h[i] = self.core.mu_to_seconds(t2 - t1)
for i in range(self.count):
inner()
return (self.h2d, self.d2h)
@kernel @kernel
def host_to_device_array(self): def test_list(self, large):
def inner():
t0 = self.core.get_rtio_counter_mu() t0 = self.core.get_rtio_counter_mu()
data = self.source_array() data = self.get_list(large)
t1 = self.core.get_rtio_counter_mu() t1 = self.core.get_rtio_counter_mu()
return 4 * len(data)/self.core.mu_to_seconds(t1-t0) self.sink(data)
t2 = self.core.get_rtio_counter_mu()
self.h2d[i] = self.core.mu_to_seconds(t1 - t0)
self.d2h[i] = self.core.mu_to_seconds(t2 - t1)
for i in range(self.count):
inner()
return (self.h2d, self.d2h)
@kernel @kernel
def host_to_device_byte_list(self): def test_array(self, large):
def inner():
t0 = self.core.get_rtio_counter_mu() t0 = self.core.get_rtio_counter_mu()
data = self.source_byte_list() data = self.get_array(large)
t1 = self.core.get_rtio_counter_mu() t1 = self.core.get_rtio_counter_mu()
return len(data)/self.core.mu_to_seconds(t1-t0) self.sink(data)
t2 = self.core.get_rtio_counter_mu()
self.h2d[i] = self.core.mu_to_seconds(t1 - t0)
self.d2h[i] = self.core.mu_to_seconds(t2 - t1)
for i in range(self.count):
inner()
return (self.h2d, self.d2h)
@kernel @kernel
def device_to_host(self): def test_async(self):
t0 = self.core.get_rtio_counter_mu() data = self.get_bytes(True)
self.sink(self.data) for _ in range(128):
t1 = self.core.get_rtio_counter_mu() self.sink_async(data)
return len(self.data)/self.core.mu_to_seconds(t1-t0) return self.get_async_throughput()
@kernel
def device_to_host_list(self):
#data = [[0]*8 for _ in range(1 << 12)]
data = [0]*(1 << 15)
t0 = self.core.get_rtio_counter_mu()
self.sink_list(data)
t1 = self.core.get_rtio_counter_mu()
return ((len(data)*4) /
self.core.mu_to_seconds(t1-t0))
@kernel
def device_to_host_array(self):
data = self.source_array()
t0 = self.core.get_rtio_counter_mu()
self.sink_array(data)
t1 = self.core.get_rtio_counter_mu()
return ((len(data)*4) /
self.core.mu_to_seconds(t1-t0))
class TransferTest(ExperimentCase): class TransferTest(ExperimentCase):
def test_host_to_device(self): @classmethod
exp = self.create(_Transfer) def setUpClass(self):
host_to_device_rate = exp.host_to_device() self.results = []
print(host_to_device_rate/(1024*1024), "MiB/s")
self.assertGreater(host_to_device_rate, 2.0e6)
def test_host_to_device_byte_list(self): @classmethod
exp = self.create(_Transfer) def tearDownClass(self):
host_to_device_rate = exp.host_to_device_byte_list() if len(self.results) == 0:
print(host_to_device_rate/(1024*1024), "MiB/s") return
self.assertGreater(host_to_device_rate, 2.0e6) max_length = max(max(len(row[0]) for row in self.results), len("Test"))
def test_host_to_device_list(self): def pad(name):
exp = self.create(_Transfer) nonlocal max_length
host_to_device_rate = exp.host_to_device_list() return name + " " * (max_length - len(name))
print(host_to_device_rate/(1024*1024), "MiB/s") print()
self.assertGreater(host_to_device_rate, 2.0e6) print("| {} | Mean (MiB/s) | std (MiB/s) |".format(pad("Test")))
print("| {} | ------------ | ------------ |".format("-" * max_length))
for v in self.results:
print("| {} | {:>12.2f} | {:>12.2f} |".format(
pad(v[0]), v[1], v[2]))
def test_host_to_device_array(self): def test_bytes_large(self):
exp = self.create(_Transfer) exp = self.create(_Transfer)
host_to_device_rate = exp.host_to_device_array() results = exp.test_bytes(True)
print(host_to_device_rate/(1024*1024), "MiB/s") host_to_device = (1 << 20) / numpy.array(results[0], numpy.float64)
self.assertGreater(host_to_device_rate, 2.0e6) device_to_host = (1 << 20) / numpy.array(results[1], numpy.float64)
host_to_device /= 1024*1024
device_to_host /= 1024*1024
self.results.append(["Bytes (1MB) H2D", host_to_device.mean(),
host_to_device.std()])
self.results.append(["Bytes (1MB) D2H", device_to_host.mean(),
device_to_host.std()])
def test_device_to_host(self): def test_bytes_small(self):
exp = self.create(_Transfer) exp = self.create(_Transfer)
device_to_host_rate = exp.device_to_host() results = exp.test_bytes(False)
print(device_to_host_rate/(1024*1024), "MiB/s") host_to_device = (1 << 10) / numpy.array(results[0], numpy.float64)
self.assertGreater(device_to_host_rate, 2.2e6) device_to_host = (1 << 10) / numpy.array(results[1], numpy.float64)
host_to_device /= 1024*1024
device_to_host /= 1024*1024
self.results.append(["Bytes (1KB) H2D", host_to_device.mean(),
host_to_device.std()])
self.results.append(["Bytes (1KB) D2H", device_to_host.mean(),
device_to_host.std()])
def test_device_to_host_list(self): def test_byte_list_large(self):
exp = self.create(_Transfer) exp = self.create(_Transfer)
rate = exp.device_to_host_list() results = exp.test_byte_list(True)
print(rate/(1024*1024), "MiB/s") host_to_device = (1 << 20) / numpy.array(results[0], numpy.float64)
self.assertGreater(rate, .15e6) device_to_host = (1 << 20) / numpy.array(results[1], numpy.float64)
host_to_device /= 1024*1024
device_to_host /= 1024*1024
self.results.append(["Bytes List (1MB) H2D", host_to_device.mean(),
host_to_device.std()])
self.results.append(["Bytes List (1MB) D2H", device_to_host.mean(),
device_to_host.std()])
def test_device_to_host_array(self): def test_byte_list_small(self):
exp = self.create(_Transfer) exp = self.create(_Transfer)
rate = exp.device_to_host_array() results = exp.test_byte_list(False)
print(rate/(1024*1024), "MiB/s") host_to_device = (1 << 10) / numpy.array(results[0], numpy.float64)
self.assertGreater(rate, .15e6) device_to_host = (1 << 10) / numpy.array(results[1], numpy.float64)
host_to_device /= 1024*1024
device_to_host /= 1024*1024
self.results.append(["Bytes List (1KB) H2D", host_to_device.mean(),
host_to_device.std()])
self.results.append(["Bytes List (1KB) D2H", device_to_host.mean(),
device_to_host.std()])
def test_list_large(self):
exp = self.create(_Transfer)
results = exp.test_list(True)
host_to_device = (1 << 20) / numpy.array(results[0], numpy.float64)
device_to_host = (1 << 20) / numpy.array(results[1], numpy.float64)
host_to_device /= 1024*1024
device_to_host /= 1024*1024
self.results.append(["I32 List (1MB) H2D", host_to_device.mean(),
host_to_device.std()])
self.results.append(["I32 List (1MB) D2H", device_to_host.mean(),
device_to_host.std()])
def test_list_small(self):
exp = self.create(_Transfer)
results = exp.test_list(False)
host_to_device = (1 << 10) / numpy.array(results[0], numpy.float64)
device_to_host = (1 << 10) / numpy.array(results[1], numpy.float64)
host_to_device /= 1024*1024
device_to_host /= 1024*1024
self.results.append(["I32 List (1KB) H2D", host_to_device.mean(),
host_to_device.std()])
self.results.append(["I32 List (1KB) D2H", device_to_host.mean(),
device_to_host.std()])
def test_array_large(self):
exp = self.create(_Transfer)
results = exp.test_array(True)
host_to_device = (1 << 20) / numpy.array(results[0], numpy.float64)
device_to_host = (1 << 20) / numpy.array(results[1], numpy.float64)
host_to_device /= 1024*1024
device_to_host /= 1024*1024
self.results.append(["I32 Array (1MB) H2D", host_to_device.mean(),
host_to_device.std()])
self.results.append(["I32 Array (1MB) D2H", device_to_host.mean(),
device_to_host.std()])
def test_array_small(self):
exp = self.create(_Transfer)
results = exp.test_array(False)
host_to_device = (1 << 10) / numpy.array(results[0], numpy.float64)
device_to_host = (1 << 10) / numpy.array(results[1], numpy.float64)
host_to_device /= 1024*1024
device_to_host /= 1024*1024
self.results.append(["I32 Array (1KB) H2D", host_to_device.mean(),
host_to_device.std()])
self.results.append(["I32 Array (1KB) D2H", device_to_host.mean(),
device_to_host.std()])
def test_async_throughput(self):
exp = self.create(_Transfer)
results = exp.test_async()
print("Async throughput: {:>6.2f}MiB/s".format(results))
class _KernelOverhead(EnvExperiment): class _KernelOverhead(EnvExperiment):
def build(self): def build(self):