242 lines
8.1 KiB
Python
Executable File
242 lines
8.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# This file is part of the MicroPython project, http://micropython.org/
|
|
# The MIT License (MIT)
|
|
# Copyright (c) 2019 Damien P. George
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import argparse
|
|
from glob import glob
|
|
|
|
sys.path.append('../tools')
|
|
import pyboard
|
|
|
|
# Paths for host executables
|
|
if os.name == 'nt':
|
|
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')
|
|
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/windows/micropython.exe')
|
|
else:
|
|
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
|
|
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython')
|
|
|
|
PYTHON_TRUTH = CPYTHON3
|
|
|
|
BENCH_SCRIPT_DIR = 'perf_bench/'
|
|
|
|
def compute_stats(lst):
|
|
avg = 0
|
|
var = 0
|
|
for x in lst:
|
|
avg += x
|
|
var += x * x
|
|
avg /= len(lst)
|
|
var = max(0, var / len(lst) - avg ** 2)
|
|
return avg, var ** 0.5
|
|
|
|
def run_script_on_target(target, script):
|
|
output = b''
|
|
err = None
|
|
|
|
if isinstance(target, pyboard.Pyboard):
|
|
# Run via pyboard interface
|
|
try:
|
|
target.enter_raw_repl()
|
|
output = target.exec_(script)
|
|
except pyboard.PyboardError as er:
|
|
err = er
|
|
else:
|
|
# Run local executable
|
|
try:
|
|
p = subprocess.run([target], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=script)
|
|
output = p.stdout
|
|
except subprocess.CalledProcessError as er:
|
|
err = er
|
|
|
|
return str(output.strip(), 'ascii'), err
|
|
|
|
def run_feature_test(target, test):
|
|
with open('feature_check/' + test + '.py', 'rb') as f:
|
|
script = f.read()
|
|
output, err = run_script_on_target(target, script)
|
|
if err is None:
|
|
return output
|
|
else:
|
|
return 'CRASH: %r' % err
|
|
|
|
def run_benchmark_on_target(target, script):
|
|
output, err = run_script_on_target(target, script)
|
|
if err is None:
|
|
time, norm, result = output.split(None, 2)
|
|
try:
|
|
return int(time), int(norm), result
|
|
except ValueError:
|
|
return -1, -1, 'CRASH: %r' % output
|
|
else:
|
|
return -1, -1, 'CRASH: %r' % err
|
|
|
|
def run_benchmarks(target, param_n, param_m, n_average, test_list):
|
|
skip_native = run_feature_test(target, 'native_check') != ''
|
|
|
|
for test_file in sorted(test_list):
|
|
print(test_file + ': ', end='')
|
|
|
|
# Check if test should be skipped
|
|
skip = skip_native and test_file.find('viper_') != -1
|
|
if skip:
|
|
print('skip')
|
|
continue
|
|
|
|
# Create test script
|
|
with open(test_file, 'rb') as f:
|
|
test_script = f.read()
|
|
with open(BENCH_SCRIPT_DIR + 'benchrun.py', 'rb') as f:
|
|
test_script += f.read()
|
|
test_script += b'bm_run(%u, %u)\n' % (param_n, param_m)
|
|
|
|
# Write full test script if needed
|
|
if 0:
|
|
with open('%s.full' % test_file, 'wb') as f:
|
|
f.write(test_script)
|
|
|
|
# Run MicroPython a given number of times
|
|
times = []
|
|
scores = []
|
|
error = None
|
|
result_out = None
|
|
for _ in range(n_average):
|
|
time, norm, result = run_benchmark_on_target(target, test_script)
|
|
if time < 0 or norm < 0:
|
|
error = result
|
|
break
|
|
if result_out is None:
|
|
result_out = result
|
|
elif result != result_out:
|
|
error = 'FAIL self'
|
|
break
|
|
times.append(time)
|
|
scores.append(1e6 * norm / time)
|
|
|
|
# Check result against truth if needed
|
|
if error is None and result_out != 'None':
|
|
_, _, result_exp = run_benchmark_on_target(PYTHON_TRUTH, test_script)
|
|
if result_out != result_exp:
|
|
error = 'FAIL truth'
|
|
break
|
|
|
|
if error is not None:
|
|
print(error)
|
|
else:
|
|
t_avg, t_sd = compute_stats(times)
|
|
s_avg, s_sd = compute_stats(scores)
|
|
print('{:.2f} {:.4f} {:.2f} {:.4f}'.format(t_avg, 100 * t_sd / t_avg, s_avg, 100 * s_sd / s_avg))
|
|
if 0:
|
|
print(' times: ', times)
|
|
print(' scores:', scores)
|
|
|
|
sys.stdout.flush()
|
|
|
|
def parse_output(filename):
|
|
with open(filename) as f:
|
|
params = f.readline()
|
|
n, m, _ = params.strip().split()
|
|
n = int(n.split('=')[1])
|
|
m = int(m.split('=')[1])
|
|
data = []
|
|
for l in f:
|
|
if l.find(': ') != -1 and l.find(': skip') == -1 and l.find('CRASH: ') == -1:
|
|
name, values = l.strip().split(': ')
|
|
values = tuple(float(v) for v in values.split())
|
|
data.append((name,) + values)
|
|
return n, m, data
|
|
|
|
def compute_diff(file1, file2, diff_score):
|
|
# Parse output data from previous runs
|
|
n1, m1, d1 = parse_output(file1)
|
|
n2, m2, d2 = parse_output(file2)
|
|
|
|
# Print header
|
|
if diff_score:
|
|
print('diff of scores (higher is better)')
|
|
else:
|
|
print('diff of microsecond times (lower is better)')
|
|
if n1 == n2 and m1 == m2:
|
|
hdr = 'N={} M={}'.format(n1, m1)
|
|
else:
|
|
hdr = 'N={} M={} vs N={} M={}'.format(n1, m1, n2, m2)
|
|
print('{:24} {:>10} -> {:>10} {:>10} {:>7}% (error%)'.format(hdr, file1, file2, 'diff', 'diff'))
|
|
|
|
# Print entries
|
|
while d1 and d2:
|
|
if d1[0][0] == d2[0][0]:
|
|
# Found entries with matching names
|
|
entry1 = d1.pop(0)
|
|
entry2 = d2.pop(0)
|
|
name = entry1[0].rsplit('/')[-1]
|
|
av1, sd1 = entry1[1 + 2 * diff_score], entry1[2 + 2 * diff_score]
|
|
av2, sd2 = entry2[1 + 2 * diff_score], entry2[2 + 2 * diff_score]
|
|
sd1 *= av1 / 100 # convert from percent sd to absolute sd
|
|
sd2 *= av2 / 100 # convert from percent sd to absolute sd
|
|
av_diff = av2 - av1
|
|
sd_diff = (sd1 ** 2 + sd2 ** 2) ** 0.5
|
|
percent = 100 * av_diff / av1
|
|
percent_sd = 100 * sd_diff / av1
|
|
print('{:24} {:10.2f} -> {:10.2f} : {:+10.2f} = {:+7.3f}% (+/-{:.2f}%)'.format(name, av1, av2, av_diff, percent, percent_sd))
|
|
elif d1[0][0] < d2[0][0]:
|
|
d1.pop(0)
|
|
else:
|
|
d2.pop(0)
|
|
|
|
def main():
|
|
cmd_parser = argparse.ArgumentParser(description='Run benchmarks for MicroPython')
|
|
cmd_parser.add_argument('-t', '--diff-time', action='store_true', help='diff time outputs from a previous run')
|
|
cmd_parser.add_argument('-s', '--diff-score', action='store_true', help='diff score outputs from a previous run')
|
|
cmd_parser.add_argument('-p', '--pyboard', action='store_true', help='run tests via pyboard.py')
|
|
cmd_parser.add_argument('-d', '--device', default='/dev/ttyACM0', help='the device for pyboard.py')
|
|
cmd_parser.add_argument('-a', '--average', default='8', help='averaging number')
|
|
cmd_parser.add_argument('N', nargs=1, help='N parameter (approximate target CPU frequency)')
|
|
cmd_parser.add_argument('M', nargs=1, help='M parameter (approximate target heap in kbytes)')
|
|
cmd_parser.add_argument('files', nargs='*', help='input test files')
|
|
args = cmd_parser.parse_args()
|
|
|
|
if args.diff_time or args.diff_score:
|
|
compute_diff(args.N[0], args.M[0], args.diff_score)
|
|
sys.exit(0)
|
|
|
|
# N, M = 50, 25 # esp8266
|
|
# N, M = 100, 100 # pyboard, esp32
|
|
# N, M = 1000, 1000 # PC
|
|
N = int(args.N[0])
|
|
M = int(args.M[0])
|
|
n_average = int(args.average)
|
|
|
|
if args.pyboard:
|
|
target = pyboard.Pyboard(args.device)
|
|
target.enter_raw_repl()
|
|
else:
|
|
target = MICROPYTHON
|
|
|
|
if len(args.files) == 0:
|
|
tests_skip = ('benchrun.py',)
|
|
if M <= 25:
|
|
# These scripts are too big to be compiled by the target
|
|
tests_skip += ('bm_chaos.py', 'bm_hexiom.py', 'misc_raytrace.py')
|
|
tests = sorted(
|
|
BENCH_SCRIPT_DIR + test_file for test_file in os.listdir(BENCH_SCRIPT_DIR)
|
|
if test_file.endswith('.py') and test_file not in tests_skip
|
|
)
|
|
else:
|
|
tests = sorted(args.files)
|
|
|
|
print('N={} M={} n_average={}'.format(N, M, n_average))
|
|
|
|
run_benchmarks(target, N, M, n_average, tests)
|
|
|
|
if isinstance(target, pyboard.Pyboard):
|
|
target.exit_raw_repl()
|
|
target.close()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|