1#!/usr/bin/env python3
2
3# This file is part of the MicroPython project, http://micropython.org/
4# The MIT License (MIT)
5# Copyright (c) 2019 Damien P. George
6
7import os
8import subprocess
9import sys
10import argparse
11from glob import glob
12
13sys.path.append("../tools")
14import pyboard
15
16# Paths for host executables
17if os.name == "nt":
18    CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3.exe")
19    MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/windows/micropython.exe")
20else:
21    CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3")
22    MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/micropython")
23
24PYTHON_TRUTH = CPYTHON3
25
26BENCH_SCRIPT_DIR = "perf_bench/"
27
28
29def compute_stats(lst):
30    avg = 0
31    var = 0
32    for x in lst:
33        avg += x
34        var += x * x
35    avg /= len(lst)
36    var = max(0, var / len(lst) - avg ** 2)
37    return avg, var ** 0.5
38
39
40def run_script_on_target(target, script):
41    output = b""
42    err = None
43
44    if isinstance(target, pyboard.Pyboard):
45        # Run via pyboard interface
46        try:
47            target.enter_raw_repl()
48            output = target.exec_(script)
49        except pyboard.PyboardError as er:
50            err = er
51    else:
52        # Run local executable
53        try:
54            p = subprocess.run(
55                target, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=script
56            )
57            output = p.stdout
58        except subprocess.CalledProcessError as er:
59            err = er
60
61    return str(output.strip(), "ascii"), err
62
63
64def run_feature_test(target, test):
65    with open("feature_check/" + test + ".py", "rb") as f:
66        script = f.read()
67    output, err = run_script_on_target(target, script)
68    if err is None:
69        return output
70    else:
71        return "CRASH: %r" % err
72
73
74def run_benchmark_on_target(target, script):
75    output, err = run_script_on_target(target, script)
76    if err is None:
77        time, norm, result = output.split(None, 2)
78        try:
79            return int(time), int(norm), result
80        except ValueError:
81            return -1, -1, "CRASH: %r" % output
82    else:
83        return -1, -1, "CRASH: %r" % err
84
85
86def run_benchmarks(target, param_n, param_m, n_average, test_list):
87    skip_complex = run_feature_test(target, "complex") != "complex"
88    skip_native = run_feature_test(target, "native_check") != ""
89
90    for test_file in sorted(test_list):
91        print(test_file + ": ", end="")
92
93        # Check if test should be skipped
94        skip = (
95            skip_complex
96            and test_file.find("bm_fft") != -1
97            or skip_native
98            and test_file.find("viper_") != -1
99        )
100        if skip:
101            print("skip")
102            continue
103
104        # Create test script
105        with open(test_file, "rb") as f:
106            test_script = f.read()
107        with open(BENCH_SCRIPT_DIR + "benchrun.py", "rb") as f:
108            test_script += f.read()
109        test_script += b"bm_run(%u, %u)\n" % (param_n, param_m)
110
111        # Write full test script if needed
112        if 0:
113            with open("%s.full" % test_file, "wb") as f:
114                f.write(test_script)
115
116        # Run MicroPython a given number of times
117        times = []
118        scores = []
119        error = None
120        result_out = None
121        for _ in range(n_average):
122            time, norm, result = run_benchmark_on_target(target, test_script)
123            if time < 0 or norm < 0:
124                error = result
125                break
126            if result_out is None:
127                result_out = result
128            elif result != result_out:
129                error = "FAIL self"
130                break
131            times.append(time)
132            scores.append(1e6 * norm / time)
133
134        # Check result against truth if needed
135        if error is None and result_out != "None":
136            _, _, result_exp = run_benchmark_on_target(PYTHON_TRUTH, test_script)
137            if result_out != result_exp:
138                error = "FAIL truth"
139
140        if error is not None:
141            print(error)
142        else:
143            t_avg, t_sd = compute_stats(times)
144            s_avg, s_sd = compute_stats(scores)
145            print(
146                "{:.2f} {:.4f} {:.2f} {:.4f}".format(
147                    t_avg, 100 * t_sd / t_avg, s_avg, 100 * s_sd / s_avg
148                )
149            )
150            if 0:
151                print("  times: ", times)
152                print("  scores:", scores)
153
154        sys.stdout.flush()
155
156
157def parse_output(filename):
158    with open(filename) as f:
159        params = f.readline()
160        n, m, _ = params.strip().split()
161        n = int(n.split("=")[1])
162        m = int(m.split("=")[1])
163        data = []
164        for l in f:
165            if l.find(": ") != -1 and l.find(": skip") == -1 and l.find("CRASH: ") == -1:
166                name, values = l.strip().split(": ")
167                values = tuple(float(v) for v in values.split())
168                data.append((name,) + values)
169    return n, m, data
170
171
172def compute_diff(file1, file2, diff_score):
173    # Parse output data from previous runs
174    n1, m1, d1 = parse_output(file1)
175    n2, m2, d2 = parse_output(file2)
176
177    # Print header
178    if diff_score:
179        print("diff of scores (higher is better)")
180    else:
181        print("diff of microsecond times (lower is better)")
182    if n1 == n2 and m1 == m2:
183        hdr = "N={} M={}".format(n1, m1)
184    else:
185        hdr = "N={} M={} vs N={} M={}".format(n1, m1, n2, m2)
186    print(
187        "{:24} {:>10} -> {:>10}   {:>10}   {:>7}% (error%)".format(
188            hdr, file1, file2, "diff", "diff"
189        )
190    )
191
192    # Print entries
193    while d1 and d2:
194        if d1[0][0] == d2[0][0]:
195            # Found entries with matching names
196            entry1 = d1.pop(0)
197            entry2 = d2.pop(0)
198            name = entry1[0].rsplit("/")[-1]
199            av1, sd1 = entry1[1 + 2 * diff_score], entry1[2 + 2 * diff_score]
200            av2, sd2 = entry2[1 + 2 * diff_score], entry2[2 + 2 * diff_score]
201            sd1 *= av1 / 100  # convert from percent sd to absolute sd
202            sd2 *= av2 / 100  # convert from percent sd to absolute sd
203            av_diff = av2 - av1
204            sd_diff = (sd1 ** 2 + sd2 ** 2) ** 0.5
205            percent = 100 * av_diff / av1
206            percent_sd = 100 * sd_diff / av1
207            print(
208                "{:24} {:10.2f} -> {:10.2f} : {:+10.2f} = {:+7.3f}% (+/-{:.2f}%)".format(
209                    name, av1, av2, av_diff, percent, percent_sd
210                )
211            )
212        elif d1[0][0] < d2[0][0]:
213            d1.pop(0)
214        else:
215            d2.pop(0)
216
217
218def main():
219    cmd_parser = argparse.ArgumentParser(description="Run benchmarks for MicroPython")
220    cmd_parser.add_argument(
221        "-t", "--diff-time", action="store_true", help="diff time outputs from a previous run"
222    )
223    cmd_parser.add_argument(
224        "-s", "--diff-score", action="store_true", help="diff score outputs from a previous run"
225    )
226    cmd_parser.add_argument(
227        "-p", "--pyboard", action="store_true", help="run tests via pyboard.py"
228    )
229    cmd_parser.add_argument(
230        "-d", "--device", default="/dev/ttyACM0", help="the device for pyboard.py"
231    )
232    cmd_parser.add_argument("-a", "--average", default="8", help="averaging number")
233    cmd_parser.add_argument(
234        "--emit", default="bytecode", help="MicroPython emitter to use (bytecode or native)"
235    )
236    cmd_parser.add_argument("N", nargs=1, help="N parameter (approximate target CPU frequency)")
237    cmd_parser.add_argument("M", nargs=1, help="M parameter (approximate target heap in kbytes)")
238    cmd_parser.add_argument("files", nargs="*", help="input test files")
239    args = cmd_parser.parse_args()
240
241    if args.diff_time or args.diff_score:
242        compute_diff(args.N[0], args.M[0], args.diff_score)
243        sys.exit(0)
244
245    # N, M = 50, 25 # esp8266
246    # N, M = 100, 100 # pyboard, esp32
247    # N, M = 1000, 1000 # PC
248    N = int(args.N[0])
249    M = int(args.M[0])
250    n_average = int(args.average)
251
252    if args.pyboard:
253        target = pyboard.Pyboard(args.device)
254        target.enter_raw_repl()
255    else:
256        target = [MICROPYTHON, "-X", "emit=" + args.emit]
257
258    if len(args.files) == 0:
259        tests_skip = ("benchrun.py",)
260        if M <= 25:
261            # These scripts are too big to be compiled by the target
262            tests_skip += ("bm_chaos.py", "bm_hexiom.py", "misc_raytrace.py")
263        tests = sorted(
264            BENCH_SCRIPT_DIR + test_file
265            for test_file in os.listdir(BENCH_SCRIPT_DIR)
266            if test_file.endswith(".py") and test_file not in tests_skip
267        )
268    else:
269        tests = sorted(args.files)
270
271    print("N={} M={} n_average={}".format(N, M, n_average))
272
273    run_benchmarks(target, N, M, n_average, tests)
274
275    if isinstance(target, pyboard.Pyboard):
276        target.exit_raw_repl()
277        target.close()
278
279
280if __name__ == "__main__":
281    main()
282