1#! /usr/bin/env python3 2 3import os 4import subprocess 5import sys 6import platform 7import argparse 8import inspect 9import re 10from glob import glob 11 12# See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0] 13# are guaranteed to always work, this one should though. 14BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None))) 15 16def base_path(*p): 17 return os.path.abspath(os.path.join(BASEPATH, *p)).replace('\\', '/') 18 19# Tests require at least CPython 3.3. If your default python3 executable 20# is of lower version, you can point MICROPY_CPYTHON3 environment var 21# to the correct executable. 22if os.name == 'nt': 23 CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe') 24 MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/windows/micropython.exe')) 25else: 26 CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3') 27 MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/unix/micropython')) 28 29# Use CPython options to not save .pyc files, to only access the core standard library 30# (not site packages which may clash with u-module names), and improve start up time. 31CPYTHON3_CMD = [CPYTHON3, "-BS"] 32 33# mpy-cross is only needed if --via-mpy command-line arg is passed 34MPYCROSS = os.getenv('MICROPY_MPYCROSS', base_path('../mpy-cross/mpy-cross')) 35 36# For diff'ing test output 37DIFF = os.getenv('MICROPY_DIFF', 'diff -u') 38 39# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale 40os.environ['PYTHONIOENCODING'] = 'utf-8' 41 42def rm_f(fname): 43 if os.path.exists(fname): 44 os.remove(fname) 45 46 47# unescape wanted regex chars and escape unwanted ones 48def convert_regex_escapes(line): 49 cs = [] 50 escape = False 51 for c in str(line, 'utf8'): 52 if escape: 53 escape = False 54 cs.append(c) 55 elif c == '\\': 56 escape = True 57 elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'): 58 cs.append('\\' + c) 59 else: 60 cs.append(c) 61 # accept carriage-return(s) before final newline 62 if cs[-1] == '\n': 63 cs[-1] = '\r*\n' 64 return bytes(''.join(cs), 'utf8') 65 66 67def run_micropython(pyb, args, test_file, is_special=False): 68 special_tests = ( 69 'micropython/meminfo.py', 'basics/bytes_compare3.py', 70 'basics/builtin_help.py', 'thread/thread_exc2.py', 71 'esp32/partition_ota.py', 72 ) 73 had_crash = False 74 if pyb is None: 75 # run on PC 76 if test_file.startswith(('cmdline/', base_path('feature_check/'))) or test_file in special_tests: 77 # special handling for tests of the unix cmdline program 78 is_special = True 79 80 if is_special: 81 # check for any cmdline options needed for this test 82 args = [MICROPYTHON] 83 with open(test_file, 'rb') as f: 84 line = f.readline() 85 if line.startswith(b'# cmdline:'): 86 # subprocess.check_output on Windows only accepts strings, not bytes 87 args += [str(c, 'utf-8') for c in line[10:].strip().split()] 88 89 # run the test, possibly with redirected input 90 try: 91 if 'repl_' in test_file: 92 # Need to use a PTY to test command line editing 93 try: 94 import pty 95 except ImportError: 96 # in case pty module is not available, like on Windows 97 return b'SKIP\n' 98 import select 99 100 def get(required=False): 101 rv = b'' 102 while True: 103 ready = select.select([master], [], [], 0.02) 104 if ready[0] == [master]: 105 rv += os.read(master, 1024) 106 else: 107 if not required or rv: 108 return rv 109 110 def send_get(what): 111 os.write(master, what) 112 return get() 113 114 with open(test_file, 'rb') as f: 115 # instead of: output_mupy = subprocess.check_output(args, stdin=f) 116 master, slave = pty.openpty() 117 p = subprocess.Popen(args, stdin=slave, stdout=slave, 118 stderr=subprocess.STDOUT, bufsize=0) 119 banner = get(True) 120 output_mupy = banner + b''.join(send_get(line) for line in f) 121 send_get(b'\x04') # exit the REPL, so coverage info is saved 122 # At this point the process might have exited already, but trying to 123 # kill it 'again' normally doesn't result in exceptions as Python and/or 124 # the OS seem to try to handle this nicely. When running Linux on WSL 125 # though, the situation differs and calling Popen.kill after the process 126 # terminated results in a ProcessLookupError. Just catch that one here 127 # since we just want the process to be gone and that's the case. 128 try: 129 p.kill() 130 except ProcessLookupError: 131 pass 132 os.close(master) 133 os.close(slave) 134 else: 135 output_mupy = subprocess.check_output(args + [test_file], stderr=subprocess.STDOUT) 136 except subprocess.CalledProcessError: 137 return b'CRASH' 138 139 else: 140 # a standard test run on PC 141 142 # create system command 143 cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit] 144 if args.heapsize is not None: 145 cmdlist.extend(['-X', 'heapsize=' + args.heapsize]) 146 147 # if running via .mpy, first compile the .py file 148 if args.via_mpy: 149 subprocess.check_output([MPYCROSS] + args.mpy_cross_flags.split() + ['-o', 'mpytest.mpy', '-X', 'emit=' + args.emit, test_file]) 150 cmdlist.extend(['-m', 'mpytest']) 151 else: 152 cmdlist.append(test_file) 153 154 # run the actual test 155 try: 156 output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT) 157 except subprocess.CalledProcessError as er: 158 had_crash = True 159 output_mupy = er.output + b'CRASH' 160 161 # clean up if we had an intermediate .mpy file 162 if args.via_mpy: 163 rm_f('mpytest.mpy') 164 165 else: 166 # run on pyboard 167 if (args.target == 'haas100'): 168 pyb.enter_raw_repl() 169 try: 170 output_mupy = pyb.execfile(test_file) 171 except haasboard.HaaSboardError as e: 172 had_crash = True 173 if not is_special and e.args[0] == 'exception': 174 output_mupy = e.args[1] + e.args[2] + b'CRASH' 175 else: 176 output_mupy = b'CRASH' 177 else: 178 pyb.enter_raw_repl() 179 try: 180 output_mupy = pyb.execfile(test_file) 181 except pyboard.PyboardError as e: 182 had_crash = True 183 if not is_special and e.args[0] == 'exception': 184 output_mupy = e.args[1] + e.args[2] + b'CRASH' 185 else: 186 output_mupy = b'CRASH' 187 188 # canonical form for all ports/platforms is to use \n for end-of-line 189 output_mupy = output_mupy.replace(b'\r\n', b'\n') 190 191 # don't try to convert the output if we should skip this test 192 if had_crash or output_mupy in (b'SKIP\n', b'CRASH'): 193 return output_mupy 194 195 if is_special or test_file in special_tests: 196 # convert parts of the output that are not stable across runs 197 with open(test_file + '.exp', 'rb') as f: 198 lines_exp = [] 199 for line in f.readlines(): 200 if line == b'########\n': 201 line = (line,) 202 else: 203 line = (line, re.compile(convert_regex_escapes(line))) 204 lines_exp.append(line) 205 lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')] 206 if output_mupy.endswith(b'\n'): 207 lines_mupy = lines_mupy[:-1] # remove erroneous last empty line 208 i_mupy = 0 209 for i in range(len(lines_exp)): 210 if lines_exp[i][0] == b'########\n': 211 # 8x #'s means match 0 or more whole lines 212 line_exp = lines_exp[i + 1] 213 skip = 0 214 while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]): 215 skip += 1 216 if i_mupy + skip >= len(lines_mupy): 217 lines_mupy[i_mupy] = b'######## FAIL\n' 218 break 219 del lines_mupy[i_mupy:i_mupy + skip] 220 lines_mupy.insert(i_mupy, b'########\n') 221 i_mupy += 1 222 else: 223 # a regex 224 if lines_exp[i][1].match(lines_mupy[i_mupy]): 225 lines_mupy[i_mupy] = lines_exp[i][0] 226 else: 227 #print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG 228 pass 229 i_mupy += 1 230 if i_mupy >= len(lines_mupy): 231 break 232 output_mupy = b''.join(lines_mupy) 233 234 return output_mupy 235 236 237def run_feature_check(pyb, args, base_path, test_file): 238 if pyb is not None and test_file.startswith("repl_"): 239 # REPL feature tests will not run via pyboard because they require prompt interactivity 240 return b"" 241 return run_micropython(pyb, args, base_path("feature_check", test_file), is_special=True) 242 243 244def run_tests(pyb, tests, args, result_dir): 245 test_count = 0 246 testcase_count = 0 247 passed_count = 0 248 failed_tests = [] 249 skipped_tests = [] 250 251 skip_tests = set() 252 skip_native = False 253 skip_int_big = False 254 skip_bytearray = False 255 skip_set_type = False 256 skip_slice = False 257 skip_async = False 258 skip_const = False 259 skip_revops = False 260 skip_io_module = False 261 skip_endian = False 262 has_complex = True 263 has_coverage = False 264 265 upy_float_precision = 32 266 267 # If we're asked to --list-tests, we can't assume that there's a 268 # connection to target, so we can't run feature checks usefully. 269 if not (args.list_tests or args.write_exp): 270 # Even if we run completely different tests in a different directory, 271 # we need to access feature_checks from the same directory as the 272 # run-tests script itself so use base_path. 273 274 # Check if micropython.native is supported, and skip such tests if it's not 275 output = run_feature_check(pyb, args, base_path, 'native_check.py') 276 if output == b'CRASH': 277 skip_native = True 278 279 # Check if arbitrary-precision integers are supported, and skip such tests if it's not 280 output = run_feature_check(pyb, args, base_path, 'int_big.py') 281 if output != b'1000000000000000000000000000000000000000000000\n': 282 skip_int_big = True 283 284 # Check if bytearray is supported, and skip such tests if it's not 285 output = run_feature_check(pyb, args, base_path, 'bytearray.py') 286 if output != b'bytearray\n': 287 skip_bytearray = True 288 289 # Check if set type (and set literals) is supported, and skip such tests if it's not 290 output = run_feature_check(pyb, args, base_path, 'set_check.py') 291 if output == b'CRASH': 292 skip_set_type = True 293 294 # Check if slice is supported, and skip such tests if it's not 295 output = run_feature_check(pyb, args, base_path, 'slice.py') 296 if output != b'slice\n': 297 skip_slice = True 298 299 # Check if async/await keywords are supported, and skip such tests if it's not 300 output = run_feature_check(pyb, args, base_path, 'async_check.py') 301 if output == b'CRASH': 302 skip_async = True 303 304 # Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not 305 output = run_feature_check(pyb, args, base_path, 'const.py') 306 if output == b'CRASH': 307 skip_const = True 308 309 # Check if __rOP__ special methods are supported, and skip such tests if it's not 310 output = run_feature_check(pyb, args, base_path, 'reverse_ops.py') 311 if output == b'TypeError\n': 312 skip_revops = True 313 314 # Check if uio module exists, and skip such tests if it doesn't 315 output = run_feature_check(pyb, args, base_path, 'uio_module.py') 316 if output != b'uio\n': 317 skip_io_module = True 318 319 # Check if emacs repl is supported, and skip such tests if it's not 320 t = run_feature_check(pyb, args, base_path, 'repl_emacs_check.py') 321 if 'True' not in str(t, 'ascii'): 322 skip_tests.add('cmdline/repl_emacs_keys.py') 323 324 # Check if words movement in repl is supported, and skip such tests if it's not 325 t = run_feature_check(pyb, args, base_path, 'repl_words_move_check.py') 326 if 'True' not in str(t, 'ascii'): 327 skip_tests.add('cmdline/repl_words_move.py') 328 329 upy_byteorder = run_feature_check(pyb, args, base_path, 'byteorder.py') 330 upy_float_precision = run_feature_check(pyb, args, base_path, 'float.py') 331 if upy_float_precision == b'CRASH': 332 upy_float_precision = 0 333 else: 334 upy_float_precision = int(upy_float_precision) 335 has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n' 336 has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n' 337 cpy_byteorder = subprocess.check_output(CPYTHON3_CMD + [base_path('feature_check/byteorder.py')]) 338 skip_endian = (upy_byteorder != cpy_byteorder) 339 340 # These tests don't test slice explicitly but rather use it to perform the test 341 misc_slice_tests = ( 342 'builtin_range', 343 'class_super', 344 'containment', 345 'errno1', 346 'fun_str', 347 'generator1', 348 'globals_del', 349 'memoryview1', 350 'memoryview_gc', 351 'object1', 352 'python34', 353 'struct_endian', 354 ) 355 356 # Some tests shouldn't be run on GitHub Actions 357 if os.getenv('GITHUB_ACTIONS') == 'true': 358 skip_tests.add('thread/stress_schedule.py') # has reliability issues 359 360 if upy_float_precision == 0: 361 skip_tests.add('extmod/uctypes_le_float.py') 362 skip_tests.add('extmod/uctypes_native_float.py') 363 skip_tests.add('extmod/uctypes_sizeof_float.py') 364 skip_tests.add('extmod/ujson_dumps_float.py') 365 skip_tests.add('extmod/ujson_loads_float.py') 366 skip_tests.add('extmod/urandom_extra_float.py') 367 skip_tests.add('misc/rge_sm.py') 368 if upy_float_precision < 32: 369 skip_tests.add('float/float2int_intbig.py') # requires fp32, there's float2int_fp30_intbig.py instead 370 skip_tests.add('float/string_format.py') # requires fp32, there's string_format_fp30.py instead 371 skip_tests.add('float/bytes_construct.py') # requires fp32 372 skip_tests.add('float/bytearray_construct.py') # requires fp32 373 if upy_float_precision < 64: 374 skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead 375 skip_tests.add('float/float2int_doubleprec_intbig.py') 376 skip_tests.add('float/float_parse_doubleprec.py') 377 378 if not has_complex: 379 skip_tests.add('float/complex1.py') 380 skip_tests.add('float/complex1_intbig.py') 381 skip_tests.add('float/complex_special_methods.py') 382 skip_tests.add('float/int_big_float.py') 383 skip_tests.add('float/true_value.py') 384 skip_tests.add('float/types.py') 385 386 if not has_coverage: 387 skip_tests.add('cmdline/cmd_parsetree.py') 388 389 # Some tests shouldn't be run on a PC 390 if args.target == 'unix': 391 # unix build does not have the GIL so can't run thread mutation tests 392 for t in tests: 393 if t.startswith('thread/mutate_'): 394 skip_tests.add(t) 395 396 # Some tests shouldn't be run on pyboard 397 if args.target != 'unix': 398 skip_tests.add('basics/exception_chain.py') # warning is not printed 399 skip_tests.add('micropython/meminfo.py') # output is very different to PC output 400 skip_tests.add('extmod/machine_mem.py') # raw memory access not supported 401 402 if args.target == 'wipy': 403 skip_tests.add('misc/print_exception.py') # requires error reporting full 404 skip_tests.update({'extmod/uctypes_%s.py' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes 405 skip_tests.add('extmod/zlibd_decompress.py') # requires zlib 406 skip_tests.add('extmod/uheapq1.py') # uheapq not supported by WiPy 407 skip_tests.add('extmod/urandom_basic.py') # requires urandom 408 skip_tests.add('extmod/urandom_extra.py') # requires urandom 409 elif args.target == 'esp8266': 410 skip_tests.add('misc/rge_sm.py') # too large 411 elif args.target == 'minimal': 412 skip_tests.add('basics/class_inplace_op.py') # all special methods not supported 413 skip_tests.add('basics/subclass_native_init.py')# native subclassing corner cases not support 414 skip_tests.add('misc/rge_sm.py') # too large 415 skip_tests.add('micropython/opt_level.py') # don't assume line numbers are stored 416 elif args.target == 'nrf': 417 skip_tests.add('basics/memoryview1.py') # no item assignment for memoryview 418 skip_tests.add('extmod/urandom_basic.py') # unimplemented: urandom.seed 419 skip_tests.add('micropython/opt_level.py') # no support for line numbers 420 skip_tests.add('misc/non_compliant.py') # no item assignment for bytearray 421 for t in tests: 422 if t.startswith('basics/io_'): 423 skip_tests.add(t) 424 elif args.target == 'qemu-arm': 425 skip_tests.add('misc/print_exception.py') # requires sys stdfiles 426 427 # Some tests are known to fail on 64-bit machines 428 if pyb is None and platform.architecture()[0] == '64bit': 429 pass 430 431 # Some tests use unsupported features on Windows 432 if os.name == 'nt': 433 skip_tests.add('import/import_file.py') # works but CPython prints forward slashes 434 435 # Some tests are known to fail with native emitter 436 # Remove them from the below when they work 437 if args.emit == 'native': 438 skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from_close generator_name'.split()}) # require raise_varargs, generator name 439 skip_tests.update({'basics/async_%s.py' % t for t in 'with with2 with_break with_return'.split()}) # require async_with 440 skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs 441 skip_tests.add('basics/annotate_var.py') # requires checking for unbound local 442 skip_tests.add('basics/del_deref.py') # requires checking for unbound local 443 skip_tests.add('basics/del_local.py') # requires checking for unbound local 444 skip_tests.add('basics/exception_chain.py') # raise from is not supported 445 skip_tests.add('basics/scope_implicit.py') # requires checking for unbound local 446 skip_tests.add('basics/try_finally_return2.py') # requires raise_varargs 447 skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local 448 skip_tests.add('extmod/uasyncio_event.py') # unknown issue 449 skip_tests.add('extmod/uasyncio_lock.py') # requires async with 450 skip_tests.add('extmod/uasyncio_micropython.py') # unknown issue 451 skip_tests.add('extmod/uasyncio_wait_for.py') # unknown issue 452 skip_tests.add('misc/features.py') # requires raise_varargs 453 skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info 454 skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native 455 skip_tests.add('micropython/emg_exc.py') # because native doesn't have proper traceback info 456 skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info 457 skip_tests.add('micropython/opt_level_lineno.py') # native doesn't have proper traceback info 458 skip_tests.add('micropython/schedule.py') # native code doesn't check pending events 459 460 for test_file in tests: 461 test_file = test_file.replace('\\', '/') 462 463 if args.filters: 464 # Default verdict is the opposit of the first action 465 verdict = "include" if args.filters[0][0] == "exclude" else "exclude" 466 for action, pat in args.filters: 467 if pat.search(test_file): 468 verdict = action 469 if verdict == "exclude": 470 continue 471 472 test_basename = test_file.replace('..', '_').replace('./', '').replace('/', '_') 473 test_name = os.path.splitext(os.path.basename(test_file))[0] 474 is_native = test_name.startswith("native_") or test_name.startswith("viper_") or args.emit == "native" 475 is_endian = test_name.endswith("_endian") 476 is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig") 477 is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray") 478 is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset") 479 is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests 480 is_async = test_name.startswith(("async_", "uasyncio_")) 481 is_const = test_name.startswith("const") 482 is_io_module = test_name.startswith("io_") 483 484 skip_it = test_file in skip_tests 485 skip_it |= skip_native and is_native 486 skip_it |= skip_endian and is_endian 487 skip_it |= skip_int_big and is_int_big 488 skip_it |= skip_bytearray and is_bytearray 489 skip_it |= skip_set_type and is_set_type 490 skip_it |= skip_slice and is_slice 491 skip_it |= skip_async and is_async 492 skip_it |= skip_const and is_const 493 skip_it |= skip_revops and "reverse_op" in test_name 494 skip_it |= skip_io_module and is_io_module 495 496 if args.list_tests: 497 if not skip_it: 498 print(test_file) 499 continue 500 501 if skip_it: 502 print("skip ", test_file) 503 skipped_tests.append(test_name) 504 continue 505 506 # get expected output 507 test_file_expected = test_file + '.exp' 508 if os.path.isfile(test_file_expected): 509 # expected output given by a file, so read that in 510 with open(test_file_expected, 'rb') as f: 511 output_expected = f.read() 512 else: 513 # run CPython to work out expected output 514 try: 515 output_expected = subprocess.check_output(CPYTHON3_CMD + [test_file]) 516 if args.write_exp: 517 with open(test_file_expected, 'wb') as f: 518 f.write(output_expected) 519 except subprocess.CalledProcessError: 520 output_expected = b'CPYTHON3 CRASH' 521 522 # canonical form for all host platforms is to use \n for end-of-line 523 output_expected = output_expected.replace(b'\r\n', b'\n') 524 525 if args.write_exp: 526 continue 527 528 # run MicroPython 529 output_mupy = run_micropython(pyb, args, test_file) 530 531 if output_mupy == b'SKIP\n': 532 print("skip ", test_file) 533 skipped_tests.append(test_name) 534 continue 535 536 testcase_count += len(output_expected.splitlines()) 537 538 filename_expected = os.path.join(result_dir, test_basename + ".exp") 539 filename_mupy = os.path.join(result_dir, test_basename + ".out") 540 541 if output_expected == output_mupy: 542 print("pass ", test_file) 543 passed_count += 1 544 rm_f(filename_expected) 545 rm_f(filename_mupy) 546 else: 547 with open(filename_expected, "wb") as f: 548 f.write(output_expected) 549 with open(filename_mupy, "wb") as f: 550 f.write(output_mupy) 551 print("FAIL ", test_file) 552 failed_tests.append(test_name) 553 554 test_count += 1 555 556 if args.list_tests: 557 return True 558 559 print("{} tests performed ({} individual testcases)".format(test_count, testcase_count)) 560 print("{} tests passed".format(passed_count)) 561 562 if len(skipped_tests) > 0: 563 print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests))) 564 if len(failed_tests) > 0: 565 print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests))) 566 return False 567 568 # all tests succeeded 569 return True 570 571 572class append_filter(argparse.Action): 573 574 def __init__(self, option_strings, dest, **kwargs): 575 super().__init__(option_strings, dest, default=[], **kwargs) 576 577 def __call__(self, parser, args, value, option): 578 if not hasattr(args, self.dest): 579 args.filters = [] 580 if option.startswith(("-e", "--e")): 581 option = "exclude" 582 else: 583 option = "include" 584 args.filters.append((option, re.compile(value))) 585 586 587def main(): 588 cmd_parser = argparse.ArgumentParser( 589 formatter_class=argparse.RawDescriptionHelpFormatter, 590 description='''Run and manage tests for MicroPython. 591 592Tests are discovered by scanning test directories for .py files or using the 593specified test files. If test files nor directories are specified, the script 594expects to be ran in the tests directory (where this file is located) and the 595builtin tests suitable for the target platform are ran. 596When running tests, run-tests compares the MicroPython output of the test with the output 597produced by running the test through CPython unless a <test>.exp file is found, in which 598case it is used as comparison. 599If a test fails, run-tests produces a pair of <test>.out and <test>.exp files in the result 600directory with the MicroPython output and the expectations, respectively. 601''', 602 epilog='''\ 603Options -i and -e can be multiple and processed in the order given. Regex 604"search" (vs "match") operation is used. An action (include/exclude) of 605the last matching regex is used: 606 run-tests -i async - exclude all, then include tests containing "async" anywhere 607 run-tests -e '/big.+int' - include all, then exclude by regex 608 run-tests -e async -i async_foo - include all, exclude async, yet still include async_foo 609''') 610 cmd_parser.add_argument('--target', default='unix', help='the target platform') 611 cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard') 612 cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device') 613 cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username') 614 cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password') 615 cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)') 616 cmd_parser.add_argument('-r', '--result-dir', default=base_path('results'), help='directory for test results') 617 cmd_parser.add_argument('-e', '--exclude', action=append_filter, metavar='REGEX', dest='filters', help='exclude test by regex on path/name.py') 618 cmd_parser.add_argument('-i', '--include', action=append_filter, metavar='REGEX', dest='filters', help='include test by regex on path/name.py') 619 cmd_parser.add_argument('--write-exp', action='store_true', help='use CPython to generate .exp files to run tests w/o CPython') 620 cmd_parser.add_argument('--list-tests', action='store_true', help='list tests instead of running them') 621 cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)') 622 cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)') 623 cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first') 624 cmd_parser.add_argument('--mpy-cross-flags', default='-mcache-lookup-bc', help='flags to pass to mpy-cross') 625 cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests') 626 cmd_parser.add_argument('files', nargs='*', help='input test files') 627 cmd_parser.add_argument('--print-failures', action='store_true', help='print the diff of expected vs. actual output for failed tests and exit') 628 cmd_parser.add_argument('--clean-failures', action='store_true', help='delete the .exp and .out files from failed tests and exit') 629 args = cmd_parser.parse_args() 630 631 if args.print_failures: 632 for exp in glob(os.path.join(args.result_dir, "*.exp")): 633 testbase = exp[:-4] 634 print() 635 print("FAILURE {0}".format(testbase)) 636 os.system("{0} {1}.exp {1}.out".format(DIFF, testbase)) 637 638 sys.exit(0) 639 640 if args.clean_failures: 641 for f in glob(os.path.join(args.result_dir, "*.exp")) + glob(os.path.join(args.result_dir, "*.out")): 642 os.remove(f) 643 644 sys.exit(0) 645 646 LOCAL_TARGETS = ('unix', 'qemu-arm',) 647 EXTERNAL_TARGETS = ('haas100', 'pyboard', 'wipy', 'esp8266', 'esp32', 'minimal', 'nrf') 648 if args.target in LOCAL_TARGETS or args.list_tests: 649 pyb = None 650 elif args.target in EXTERNAL_TARGETS: 651 if (args.target == 'haas100'): 652 global haasboard 653 sys.path.append(base_path('../engine/tools')) 654 import haasboard 655 pyb = haasboard.HaaSboard(args.device, args.baudrate, args.user, args.password) 656 pyb.enter_raw_repl() 657 else: 658 global pyboard 659 sys.path.append(base_path('../engine/tools')) 660 import pyboard 661 pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) 662 pyb.enter_raw_repl() 663 else: 664 raise ValueError('target must be one of %s' % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS)) 665 666 if len(args.files) == 0: 667 if args.test_dirs is None: 668 test_dirs = ('basics', 'micropython', 'misc', 'extmod',) 669 if args.target == 'pyboard': 670 # run pyboard tests 671 test_dirs += ('float', 'stress', 'pyb', 'pybnative', 'inlineasm') 672 elif args.target in ('esp8266', 'esp32', 'minimal', 'nrf'): 673 test_dirs += ('float',) 674 elif args.target == 'wipy': 675 # run WiPy tests 676 test_dirs += ('wipy',) 677 elif args.target == 'unix': 678 # run PC tests 679 test_dirs += ('float', 'import', 'io', 'stress', 'unicode', 'unix', 'cmdline',) 680 elif args.target == 'qemu-arm': 681 if not args.write_exp: 682 raise ValueError('--target=qemu-arm must be used with --write-exp') 683 # Generate expected output files for qemu run. 684 # This list should match the test_dirs tuple in tinytest-codegen.py. 685 test_dirs += ('float', 'inlineasm', 'qemu-arm',) 686 else: 687 # run tests from these directories 688 test_dirs = args.test_dirs 689 tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files) 690 else: 691 # tests explicitly given 692 tests = args.files 693 694 if not args.keep_path: 695 # clear search path to make sure tests use only builtin modules and those in extmod 696 os.environ['MICROPYPATH'] = os.pathsep + base_path('../extmod') 697 698 try: 699 os.makedirs(args.result_dir, exist_ok=True) 700 res = run_tests(pyb, tests, args, args.result_dir) 701 finally: 702 if pyb: 703 pyb.exit_raw_repl() 704 pyb.exit_python_mode() 705 pyb.close() 706 707 if not res: 708 sys.exit(1) 709 710if __name__ == "__main__": 711 main() 712