1#!/usr/bin/env python3 2# vim: set syntax=python ts=4 : 3# 4# Copyright (c) 2018-2025 Intel Corporation 5# Copyright 2022 NXP 6# Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved. 7# 8# SPDX-License-Identifier: Apache-2.0 9 10import argparse 11import json 12import logging 13import os 14import re 15import shutil 16import subprocess 17import sys 18from collections.abc import Generator 19from datetime import datetime, timezone 20from importlib import metadata 21from pathlib import Path 22 23import zephyr_module 24from twisterlib.constants import SUPPORTED_SIMS 25from twisterlib.coverage import supported_coverage_formats 26from twisterlib.error import TwisterRuntimeError 27from twisterlib.log_helper import log_command 28 29logger = logging.getLogger('twister') 30 31ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") 32if not ZEPHYR_BASE: 33 sys.exit("$ZEPHYR_BASE environment variable undefined") 34 35# Use this for internal comparisons; that's what canonicalization is 36# for. Don't use it when invoking other components of the build system 37# to avoid confusing and hard to trace inconsistencies in error messages 38# and logs, generated Makefiles, etc. compared to when users invoke these 39# components directly. 40# Note "normalization" is different from canonicalization, see os.path. 41canonical_zephyr_base = os.path.realpath(ZEPHYR_BASE) 42 43 44def _get_installed_packages() -> Generator[str, None, None]: 45 """Return list of installed python packages.""" 46 for dist in metadata.distributions(): 47 yield dist.metadata['Name'] 48 49 50def python_version_guard(): 51 min_ver = (3, 10) 52 if sys.version_info < min_ver: 53 min_ver_str = '.'.join([str(v) for v in min_ver]) 54 cur_ver_line = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" 55 print(f"Unsupported Python version {cur_ver_line}.") 56 print(f"Currently, Twister requires at least Python {min_ver_str}.") 57 print("Install a newer Python version and retry.") 58 sys.exit(1) 59 60 61installed_packages: list[str] = list(_get_installed_packages()) 62PYTEST_PLUGIN_INSTALLED = 'pytest-twister-harness' in installed_packages 63 64 65def norm_path(astring): 66 newstring = os.path.normpath(astring).replace(os.sep, '/') 67 return newstring 68 69 70def add_parse_arguments(parser = None) -> argparse.ArgumentParser: 71 if parser is None: 72 parser = argparse.ArgumentParser( 73 description=__doc__, 74 formatter_class=argparse.RawDescriptionHelpFormatter, 75 allow_abbrev=False) 76 parser.fromfile_prefix_chars = "+" 77 78 case_select = parser.add_argument_group("Test case selection", 79 """ 80Artificially long but functional example: 81 $ ./scripts/twister -v \\ 82 --testsuite-root tests/ztest/base \\ 83 --testsuite-root tests/kernel \\ 84 --test tests/ztest/base/testing.ztest.verbose_0 \\ 85 --test tests/kernel/fifo/fifo_api/kernel.fifo 86 87 "kernel.fifo.poll" is one of the test section names in 88 __/fifo_api/testcase.yaml 89 """) 90 91 test_plan_report = parser.add_argument_group( 92 title="Test plan reporting", 93 description="Report the composed test plan details and exit (dry-run)." 94 ) 95 96 test_plan_report_xor = test_plan_report.add_mutually_exclusive_group() 97 98 platform_group_option = parser.add_mutually_exclusive_group() 99 100 run_group_option = parser.add_mutually_exclusive_group() 101 102 device = parser.add_mutually_exclusive_group() 103 104 test_or_build = parser.add_mutually_exclusive_group() 105 106 test_xor_subtest = case_select.add_mutually_exclusive_group() 107 108 test_xor_generator = case_select.add_mutually_exclusive_group() 109 110 valgrind_asan_group = parser.add_mutually_exclusive_group() 111 112 footprint_group = parser.add_argument_group( 113 title="Memory footprint", 114 description="Collect and report ROM/RAM size footprint for the test instance images built.") 115 116 coverage_group = parser.add_argument_group( 117 title="Code coverage", 118 description="Build with code coverage support, collect code coverage statistics " 119 "executing tests, compose code coverage report at the end.\n" 120 "Effective for devices with 'HAS_COVERAGE_SUPPORT'.") 121 122 test_plan_report_xor.add_argument( 123 "-E", 124 "--save-tests", 125 metavar="FILENAME", 126 action="store", 127 help="Write a list of tests and platforms to be run to %(metavar)s file and stop execution." 128 " The resulting file will have the same content as 'testplan.json'." 129 ) 130 131 case_select.add_argument( 132 "-F", 133 "--load-tests", 134 metavar="FILENAME", 135 action="store", 136 help="Load a list of tests and platforms to be run " 137 "from a JSON file ('testplan.json' schema)." 138 ) 139 140 case_select.add_argument( 141 "-T", "--testsuite-root", action="append", default=[], type = norm_path, 142 help="Base directory to recursively search for test cases. All " 143 "testcase.yaml files under here will be processed. May be " 144 "called multiple times. Defaults to the 'samples/' and " 145 "'tests/' directories at the base of the Zephyr tree.") 146 147 case_select.add_argument( 148 "-f", 149 "--only-failed", 150 action="store_true", 151 help="Run only those tests that failed the previous twister run " 152 "invocation.") 153 154 test_plan_report_xor.add_argument("--list-tests", action="store_true", 155 help="""List of all sub-test functions recursively found in 156 all --testsuite-root arguments. The output is flattened and reports detailed 157 sub-test names without their directories. 158 Note: sub-test names can share the same test scenario identifier prefix 159 (section.subsection) even if they are from different test projects. 160 """) 161 162 test_plan_report_xor.add_argument("--test-tree", action="store_true", 163 help="""Output the test plan in a tree form.""") 164 165 platform_group_option.add_argument( 166 "-G", 167 "--integration", 168 action="store_true", 169 help="Run integration tests") 170 171 platform_group_option.add_argument( 172 "--emulation-only", action="store_true", 173 help="Only build and run emulation platforms") 174 175 run_group_option.add_argument( 176 "--device-testing", action="store_true", 177 help="Test on device directly. Specify the serial device to " 178 "use with the --device-serial option.") 179 180 run_group_option.add_argument("--generate-hardware-map", 181 help="""Probe serial devices connected to this platform 182 and create a hardware map file to be used with 183 --device-testing 184 """) 185 186 run_group_option.add_argument( 187 "--simulation", dest="sim_name", choices=SUPPORTED_SIMS, 188 help="Selects which simulation to use. Must match one of the names defined in the board's " 189 "manifest. If multiple simulator are specified in the selected board and this " 190 "argument is not passed, then the first simulator is selected.") 191 192 193 device.add_argument("--device-serial", 194 help="""Serial device for accessing the board 195 (e.g., /dev/ttyACM0) 196 """) 197 198 device.add_argument("--device-serial-pty", 199 help="""Script for controlling pseudoterminal. 200 Twister believes that it interacts with a terminal 201 when it actually interacts with the script. 202 203 E.g "twister --device-testing 204 --device-serial-pty <script> 205 """) 206 207 device.add_argument("--hardware-map", 208 help="""Load hardware map from a file. This will be used 209 for testing on hardware that is listed in the file. 210 """) 211 212 parser.add_argument("--device-flash-timeout", type=int, default=60, 213 help="""Set timeout for the device flash operation in seconds. 214 """) 215 216 parser.add_argument("--device-flash-with-test", action="store_true", 217 help="""Add a test case timeout to the flash operation timeout 218 when flash operation also executes test case on the platform. 219 """) 220 221 parser.add_argument("--flash-before", action="store_true", default=False, 222 help="""Flash device before attaching to serial port. 223 This is useful for devices that share the same port for programming 224 and serial console, or use soft-USB, where flash must come first. 225 Also, it skips reading remaining logs from the old image run. 226 """) 227 228 test_or_build.add_argument( 229 "-b", 230 "--build-only", 231 action="store_true", 232 default="--prep-artifacts-for-testing" in sys.argv, 233 help="Only build the code, do not attempt to run the code on targets." 234 ) 235 236 test_or_build.add_argument( 237 "--prep-artifacts-for-testing", action="store_true", 238 help="Generate artifacts for testing, do not attempt to run the " 239 "code on targets.") 240 241 parser.add_argument( 242 "--package-artifacts", 243 help="Package artifacts needed for flashing in a file to be used with --test-only" 244 ) 245 246 test_or_build.add_argument( 247 "--test-only", action="store_true", 248 help="""Only run device tests with current artifacts, do not build 249 the code""") 250 251 parser.add_argument("--timeout-multiplier", type=float, default=1, 252 help="""Globally adjust tests timeouts by specified multiplier. The resulting test 253 timeout would be multiplication of test timeout value, board-level timeout multiplier 254 and global timeout multiplier (this parameter)""") 255 256 parser.add_argument( 257 "--test-pattern", action="append", 258 help="""Run only the tests matching the specified pattern. The pattern 259 can include regular expressions. 260 """) 261 262 test_xor_subtest.add_argument( 263 "-s", "--test", "--scenario", action="append", type = norm_path, 264 help="""Run only the specified test suite scenario. These are named by 265 'path/relative/to/Zephyr/base/section.subsection_in_testcase_yaml', 266 or just 'section.subsection' identifier. With '--testsuite-root' option 267 the scenario will be found faster. 268 """) 269 270 test_xor_subtest.add_argument( 271 "--sub-test", action="append", 272 help="""Recursively find sub-test functions (test cases) and run the entire 273 test scenario (section.subsection) where they were found, including all sibling test 274 functions. Sub-tests are named by: 275 'section.subsection_in_testcase_yaml.ztest_suite.ztest_without_test_prefix'. 276 Example_1: 'kernel.fifo.fifo_api_1cpu.fifo_loop' where 'kernel.fifo' is a test scenario 277 name (section.subsection) and 'fifo_api_1cpu.fifo_loop' is a Ztest 'suite_name.test_name'. 278 Example_2: 'debug.coredump.logging_backend' is a standalone test scenario name. 279 Note: This selection mechanism works only for Ztest suite and test function names in 280 the source files which are not generated by macro-substitutions. 281 Note: With --no-detailed-test-id use only Ztest names without scenario name. 282 """) 283 284 parser.add_argument( 285 "--pytest-args", action="append", 286 help="""Pass additional arguments to the pytest subprocess. This parameter 287 will extend the pytest_args from the harness_config in YAML file. 288 """) 289 290 parser.add_argument( 291 "--ctest-args", action="append", 292 help="""Pass additional arguments to the ctest subprocess. This parameter 293 will extend the ctest_args from the harness_config in YAML file. 294 """) 295 296 valgrind_asan_group.add_argument( 297 "--enable-valgrind", action="store_true", 298 help="""Run binary through valgrind and check for several memory access 299 errors. Valgrind needs to be installed on the host. This option only 300 works with host binaries such as those generated for the native_sim 301 configuration and is mutual exclusive with --enable-asan. 302 """) 303 304 valgrind_asan_group.add_argument( 305 "--enable-asan", action="store_true", 306 help="""Enable address sanitizer to check for several memory access 307 errors. Libasan needs to be installed on the host. This option only 308 works with host binaries such as those generated for the native_sim 309 configuration and is mutual exclusive with --enable-valgrind. 310 """) 311 312 # Start of individual args place them in alpha-beta order 313 314 board_root_list = [f"{ZEPHYR_BASE}/boards", f"{ZEPHYR_BASE}/subsys/testsuite/boards"] 315 316 modules = zephyr_module.parse_modules(ZEPHYR_BASE) 317 for module in modules: 318 board_root = module.meta.get("build", {}).get("settings", {}).get("board_root") 319 if board_root: 320 board_root_list.append(os.path.join(module.project, board_root, "boards")) 321 322 parser.add_argument( 323 "-A", "--board-root", action="append", default=board_root_list, 324 help="""Directory to search for board configuration files. All .yaml 325files in the directory will be processed. The directory should have the same 326structure in the main Zephyr tree: boards/<vendor>/<board_name>/""") 327 328 parser.add_argument( 329 "--allow-installed-plugin", action="store_true", default=None, 330 help="Allow to use pytest plugin installed by pip for pytest tests." 331 ) 332 333 parser.add_argument( 334 "-a", "--arch", action="append", 335 help="Arch filter for testing. Takes precedence over --platform. " 336 "If unspecified, test all arches. Multiple invocations " 337 "are treated as a logical 'or' relationship") 338 339 parser.add_argument( 340 "-B", "--subset", 341 help="Only run a subset of the tests, 1/4 for running the first 25%%, " 342 "3/5 means run the 3rd fifth of the total. " 343 "This option is useful when running a large number of tests on " 344 "different hosts to speed up execution time.") 345 346 parser.add_argument( 347 "--shuffle-tests", action="store_true", default=None, 348 help="""Shuffle test execution order to get randomly distributed tests across subsets. 349 Used only when --subset is provided.""") 350 351 parser.add_argument( 352 "--shuffle-tests-seed", action="store", default=None, 353 help="""Seed value for random generator used to shuffle tests. 354 If not provided, seed in generated by system. 355 Used only when --shuffle-tests is provided.""") 356 357 parser.add_argument( 358 "-c", "--clobber-output", action="store_true", 359 help="Cleaning the output directory will simply delete it instead " 360 "of the default policy of renaming.") 361 362 parser.add_argument( 363 "--cmake-only", action="store_true", 364 help="Only run cmake, do not build or run.") 365 366 coverage_group.add_argument("--enable-coverage", action="store_true", 367 help="Enable code coverage collection using gcov.") 368 369 coverage_group.add_argument("-C", "--coverage", action="store_true", 370 help="Generate coverage reports. Implies " 371 "--enable-coverage to collect the coverage data first.") 372 373 coverage_group.add_argument("--gcov-tool", type=Path, default=None, 374 help="Path to the 'gcov' tool to use for code coverage reports. " 375 "By default it will be chosen in the following order:" 376 " using ZEPHYR_TOOLCHAIN_VARIANT ('llvm': from LLVM_TOOLCHAIN_PATH)," 377 " gcov installed on the host - for 'native' or 'unit' platform types," 378 " using ZEPHYR_SDK_INSTALL_DIR.") 379 380 coverage_group.add_argument("--coverage-basedir", default=ZEPHYR_BASE, 381 help="Base source directory for coverage report.") 382 383 coverage_group.add_argument("--coverage-platform", action="append", default=[], 384 help="Platforms to run coverage reports on. " 385 "This option may be used multiple times. " 386 "Default to what was selected with --platform.") 387 388 coverage_group.add_argument("--coverage-tool", choices=['lcov', 'gcovr'], default='gcovr', 389 help="Tool to use to generate coverage reports (%(default)s - default).") 390 391 coverage_group.add_argument("--coverage-formats", action="store", default=None, 392 help="Output formats to use for generated coverage reports " + 393 "as a comma-separated list without spaces. " + 394 "Valid options for 'gcovr' tool are: " + 395 ','.join(supported_coverage_formats['gcovr']) + " (html - default)." + 396 " Valid options for 'lcov' tool are: " + 397 ','.join(supported_coverage_formats['lcov']) + " (html,lcov - default).") 398 399 coverage_group.add_argument("--coverage-per-instance", action="store_true", default=False, 400 help="""Compose individual coverage reports for each test suite 401 when coverage reporting is enabled; it might run in addition to 402 the default aggregation mode which produces one coverage report for 403 all executed test suites. Default: %(default)s""") 404 405 coverage_group.add_argument("--disable-coverage-aggregation", 406 action="store_true", default=False, 407 help="""Don't aggregate coverage report statistics for all the test suites 408 selected to run with enabled coverage. Requires another reporting mode to be 409 active (`--coverage-per-instance`) to have at least one of these reporting 410 modes. Default: %(default)s""") 411 412 parser.add_argument( 413 "--test-config", 414 action="store", 415 default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"), 416 help="Path to file with plans and test configurations." 417 ) 418 419 parser.add_argument("--level", action="store", 420 help="Test level to be used. By default, no levels are used for filtering " 421 "and do the selection based on existing filters.") 422 423 parser.add_argument( 424 "--device-serial-baud", action="store", default=None, 425 help="Serial device baud rate (default 115200)") 426 427 parser.add_argument( 428 "--disable-suite-name-check", action="store_true", default=False, 429 help="Disable extended test suite name verification at the beginning " 430 "of Ztest test. This option could be useful for tests or " 431 "platforms, which from some reasons cannot print early logs.") 432 433 parser.add_argument("-e", "--exclude-tag", action="append", 434 help="Specify tags of tests that should not run. " 435 "Default is to run all tests with all tags.") 436 437 parser.add_argument( 438 "--enable-lsan", action="store_true", 439 help="""Enable leak sanitizer to check for heap memory leaks. 440 Libasan needs to be installed on the host. This option only 441 works with host binaries such as those generated for the native_sim 442 configuration and when --enable-asan is given. 443 """) 444 445 parser.add_argument( 446 "--enable-ubsan", action="store_true", 447 help="""Enable undefined behavior sanitizer to check for undefined 448 behaviour during program execution. It uses an optional runtime library 449 to provide better error diagnostics. This option only works with host 450 binaries such as those generated for the native_sim configuration. 451 """) 452 453 parser.add_argument( 454 "--filter", choices=['buildable', 'runnable'], 455 default='runnable' if "--device-testing" in sys.argv else 'buildable', 456 help="""Filter tests to be built and executed. By default everything is 457 built and if a test is runnable (emulation or a connected device), it 458 is run. This option allows for example to only build tests that can 459 actually be run. Runnable is a subset of buildable.""") 460 461 parser.add_argument("--force-color", action="store_true", 462 help="Always output ANSI color escape sequences " 463 "even when the output is redirected (not a tty)") 464 465 parser.add_argument("--force-toolchain", action="store_true", 466 help="Do not filter based on toolchain, use the set " 467 " toolchain unconditionally") 468 469 footprint_group.add_argument( 470 "--create-rom-ram-report", 471 action="store_true", 472 help="Generate detailed json reports with ROM/RAM symbol sizes for each test image built " 473 "using additional build option `--target footprint`.") 474 475 footprint_group.add_argument( 476 "--footprint-report", 477 nargs="?", 478 default=None, 479 choices=['all', 'ROM', 'RAM'], 480 const="all", 481 help="Select which memory area symbols' data to collect as 'footprint' property " 482 "of each test suite built, and report in 'twister_footprint.json' together " 483 "with the relevant execution metadata the same way as in `twister.json`. " 484 "Implies '--create-rom-ram-report' to generate the footprint data files. " 485 "No value means '%(const)s'. Default: %(default)s""") 486 487 footprint_group.add_argument( 488 "--enable-size-report", 489 action="store_true", 490 help="Collect and report ROM/RAM section sizes for each test image built.") 491 492 footprint_group.add_argument( 493 "--footprint-from-buildlog", 494 action = "store_true", 495 help="Take ROM/RAM sections footprint summary values from the 'build.log' " 496 "instead of 'objdump' results used otherwise." 497 "Requires --enable-size-report or one of the baseline comparison modes." 498 "Warning: the feature will not work correctly with sysbuild.") 499 500 compare_group_option = footprint_group.add_mutually_exclusive_group() 501 502 compare_group_option.add_argument( 503 "-m", "--last-metrics", 504 action="store_true", 505 help="Compare footprints to the previous twister invocation as a baseline " 506 "running in the same output directory. " 507 "Implies --enable-size-report option.") 508 509 compare_group_option.add_argument( 510 "--compare-report", 511 help="Use this report file as a baseline for footprint comparison. " 512 "The file should be of 'twister.json' schema. " 513 "Implies --enable-size-report option.") 514 515 footprint_group.add_argument( 516 "--show-footprint", 517 action="store_true", 518 help="With footprint comparison to a baseline, log ROM/RAM section deltas. ") 519 520 footprint_group.add_argument( 521 "-H", "--footprint-threshold", 522 type=float, 523 default=5.0, 524 help="With footprint comparison to a baseline, " 525 "warn the user for any of the footprint metric change which is greater or equal " 526 "to the specified percentage value. " 527 "Default is %(default)s for %(default)s%% delta from the new footprint value. " 528 "Use zero to warn on any footprint metric increase.") 529 530 footprint_group.add_argument( 531 "-D", "--all-deltas", 532 action="store_true", 533 help="With footprint comparison to a baseline, " 534 "warn on any footprint change, increase or decrease. " 535 "Implies --footprint-threshold=0") 536 537 footprint_group.add_argument( 538 "-z", "--size", 539 action="append", 540 metavar='FILENAME', 541 help="Ignore all other command line options and just produce a report to " 542 "stdout with ROM/RAM section sizes on the specified binary images.") 543 544 parser.add_argument( 545 "-i", "--inline-logs", action="store_true", 546 help="Upon test failure, print relevant log data to stdout " 547 "instead of just a path to it.") 548 549 parser.add_argument("--ignore-platform-key", action="store_true", 550 help="Do not filter based on platform key") 551 552 parser.add_argument( 553 "-j", "--jobs", type=int, 554 help="Number of jobs for building, defaults to number of CPU threads, " 555 "overcommitted by factor 2 when --build-only.") 556 557 parser.add_argument( 558 "-K", "--force-platform", action="store_true", 559 help="""Force testing on selected platforms, 560 even if they are excluded in the test configuration (testcase.yaml).""" 561 ) 562 563 parser.add_argument( 564 "-l", "--all", action="store_true", 565 help="Build/test on all platforms. Any --platform arguments " 566 "ignored.") 567 568 test_plan_report_xor.add_argument("--list-tags", action="store_true", 569 help="List all tags occurring in selected tests.") 570 571 parser.add_argument("--log-file", metavar="FILENAME", action="store", 572 help="Specify a file where to save logs.") 573 574 parser.add_argument( 575 "-M", "--runtime-artifact-cleanup", choices=['pass', 'all'], 576 default=None, const='pass', nargs='?', 577 help="""Cleanup test artifacts. The default behavior is 'pass' 578 which only removes artifacts of passing tests. If you wish to 579 remove all artificats including those of failed tests, use 'all'.""") 580 581 parser.add_argument( 582 "--keep-artifacts", action="append", default=[], 583 help="""Keep specified artifacts when cleaning up at runtime. Multiple invocations 584 are possible.""" 585 ) 586 test_xor_generator.add_argument( 587 "-N", "--ninja", action="store_true", 588 default=not any(a in sys.argv for a in ("-k", "--make")), 589 help="Use the Ninja generator with CMake. (This is the default)") 590 591 test_xor_generator.add_argument( 592 "-k", "--make", action="store_true", 593 help="Use the unix Makefile generator with CMake.") 594 595 parser.add_argument( 596 "-n", "--no-clean", action="store_true", 597 help="Re-use the outdir before building. Will result in " 598 "faster compilation since builds will be incremental.") 599 600 parser.add_argument( 601 "--aggressive-no-clean", action="store_true", 602 help="Re-use the outdir before building and do not re-run cmake. Will result in " 603 "much faster compilation since builds will be incremental. This option might " 604 " result in build failures and inconsistencies if dependencies change or when " 605 " applied on a significantly changed code base. Use on your own " 606 " risk. It is recommended to only use this option for local " 607 " development and when testing localized change in a subsystem.") 608 609 parser.add_argument( 610 '--detailed-test-id', action='store_true', 611 help="Compose each test Suite name from its configuration path (relative to root) and " 612 "the appropriate Scenario name using PATH_TO_TEST_CONFIG/SCENARIO_NAME schema. " 613 "Also (for Ztest only), prefix each test Case name with its Scenario name. " 614 "For example: 'kernel.common.timing' Scenario with test Suite name " 615 "'tests/kernel/sleep/kernel.common.timing' and 'kernel.common.timing.sleep.usleep' " 616 "test Case (where 'sleep' is its Ztest suite name and 'usleep' is Ztest test name.") 617 618 parser.add_argument( 619 "--no-detailed-test-id", dest='detailed_test_id', action="store_false", 620 help="Don't prefix each test Suite name with its configuration path, " 621 "so it is the same as the appropriate Scenario name. " 622 "Also (for Ztest only), don't prefix each Ztest Case name with its Scenario name. " 623 "For example: 'kernel.common.timing' Scenario name, the same Suite name, " 624 "and 'sleep.usleep' test Case (where 'sleep' is its Ztest suite name " 625 "and 'usleep' is Ztest test name.") 626 627 # Do not include paths in names by default. 628 parser.set_defaults(detailed_test_id=False) 629 630 parser.add_argument( 631 "--detailed-skipped-report", action="store_true", 632 help="Generate a detailed report with all skipped test cases " 633 "including those that are filtered based on testsuite definition." 634 ) 635 636 parser.add_argument( 637 "-O", "--outdir", 638 default=os.path.join(os.getcwd(), "twister-out"), 639 help="Output directory for logs and binaries. " 640 "Default is 'twister-out' in the current directory. " 641 "This directory will be cleaned unless '--no-clean' is set. " 642 "The '--clobber-output' option controls what cleaning does.") 643 644 parser.add_argument( 645 "-o", "--report-dir", 646 help="""Output reports containing results of the test run into the 647 specified directory. 648 The output will be both in JSON and JUNIT format 649 (twister.json and twister.xml). 650 """) 651 652 parser.add_argument("--overflow-as-errors", action="store_true", 653 help="Treat RAM/SRAM overflows as errors.") 654 655 parser.add_argument("--report-filtered", action="store_true", 656 help="Include filtered tests in the reports.") 657 658 parser.add_argument("-P", "--exclude-platform", action="append", default=[], 659 help="""Exclude platforms and do not build or run any tests 660 on those platforms. This option can be called multiple times. 661 """ 662 ) 663 664 parser.add_argument("--persistent-hardware-map", action='store_true', 665 help="""With --generate-hardware-map, tries to use 666 persistent names for serial devices on platforms 667 that support this feature (currently only Linux). 668 """) 669 670 parser.add_argument( 671 "--vendor", action="append", default=[], 672 help="Vendor filter for testing") 673 674 parser.add_argument( 675 "-p", "--platform", action="append", default=[], 676 help="Platform filter for testing. This option may be used multiple " 677 "times. Test suites will only be built/run on the platforms " 678 "specified. If this option is not used, then platforms marked " 679 "as default in the platform metadata file will be chosen " 680 "to build and test. ") 681 parser.add_argument( 682 "--platform-pattern", action="append", default=[], 683 help="""Platform regular expression filter for testing. This option may be used multiple 684 times. Test suites will only be built/run on the platforms 685 matching the specified patterns. If this option is not used, then platforms marked 686 as default in the platform metadata file will be chosen 687 to build and test. 688 """) 689 690 parser.add_argument( 691 "--platform-reports", action="store_true", 692 help="""Create individual reports for each platform. 693 """) 694 695 parser.add_argument("--pre-script", 696 help="""specify a pre script. This will be executed 697 before device handler open serial port and invoke runner. 698 """) 699 700 parser.add_argument( 701 "--quarantine-list", 702 action="append", 703 metavar="FILENAME", 704 help="Load list of test scenarios under quarantine. The entries in " 705 "the file need to correspond to the test scenarios names as in " 706 "corresponding tests .yaml files. These scenarios " 707 "will be skipped with quarantine as the reason.") 708 709 parser.add_argument( 710 "--quarantine-verify", 711 action="store_true", 712 help="Use the list of test scenarios under quarantine and run them " 713 "to verify their current status.") 714 715 parser.add_argument( 716 "--quit-on-failure", 717 action="store_true", 718 help="""quit twister once there is build / run failure 719 """) 720 721 parser.add_argument( 722 "--report-name", 723 help="""Create a report with a custom name. 724 """) 725 726 parser.add_argument( 727 "--report-summary", action="store", nargs='?', type=int, const=0, 728 help="Show failed/error report from latest run. Default shows all items found. " 729 "However, you can specify the number of items (e.g. --report-summary 15). " 730 "It also works well with the --outdir switch.") 731 732 parser.add_argument( 733 "--report-suffix", 734 help="""Add a suffix to all generated file names, for example to add a 735 version or a commit ID. 736 """) 737 738 parser.add_argument( 739 "--report-all-options", action="store_true", 740 help="""Show all command line options applied, including defaults, as 741 environment.options object in twister.json. Default: show only non-default settings. 742 """) 743 744 parser.add_argument( 745 "--retry-failed", type=int, default=0, 746 help="Retry failing tests again, up to the number of times specified.") 747 748 parser.add_argument( 749 "--retry-interval", type=int, default=60, 750 help="Retry failing tests after specified period of time.") 751 752 parser.add_argument( 753 "--retry-build-errors", action="store_true", 754 help="Retry build errors as well.") 755 756 parser.add_argument( 757 "-S", "--enable-slow", action="store_true", 758 default="--enable-slow-only" in sys.argv, 759 help="Execute time-consuming test cases that have been marked " 760 "as 'slow' in testcase.yaml. Normally these are only built.") 761 762 parser.add_argument( 763 "--enable-slow-only", action="store_true", 764 help="Execute time-consuming test cases that have been marked " 765 "as 'slow' in testcase.yaml only. This also set the option --enable-slow") 766 767 parser.add_argument( 768 "--seed", type=int, 769 help="Seed for native_sim pseudo-random number generator") 770 771 parser.add_argument( 772 "--short-build-path", 773 action="store_true", 774 help="Create shorter build directory paths based on symbolic links. " 775 "The shortened build path will be used by CMake for generating " 776 "the build system and executing the build. Use this option if " 777 "you experience build failures related to path length, for " 778 "example on Windows OS. This option can be used only with " 779 "'--ninja' argument (to use Ninja build generator).") 780 781 parser.add_argument( 782 "-t", "--tag", action="append", 783 help="Specify tags to restrict which tests to run by tag value. " 784 "Default is to not do any tag filtering. Multiple invocations " 785 "are treated as a logical 'or' relationship.") 786 787 parser.add_argument("--timestamps", 788 action="store_true", 789 help="Print all messages with time stamps.") 790 791 parser.add_argument( 792 "-u", 793 "--no-update", 794 action="store_true", 795 help="Do not update the results of the last run. This option " 796 "is only useful when reusing the same output directory of " 797 "twister, for example when re-running failed tests with --only-failed " 798 "or --no-clean. This option is for debugging purposes only.") 799 800 parser.add_argument( 801 "-v", 802 "--verbose", 803 action="count", 804 default=0, 805 help="Call multiple times to increase verbosity.") 806 807 parser.add_argument( 808 "-ll", 809 "--log-level", 810 type=str.upper, 811 default='INFO', 812 choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], 813 help="Select the threshold event severity for which you'd like to receive logs in console." 814 " Default is INFO.") 815 816 parser.add_argument("-W", "--disable-warnings-as-errors", action="store_true", 817 help="Do not treat warning conditions as errors.") 818 819 parser.add_argument( 820 "--west-flash", nargs='?', const=[], 821 help="""Comma separated list of additional flags passed to west when 822 running with --device-testing. 823 824 E.g "twister --device-testing --device-serial /dev/ttyACM0 825 --west-flash="--board-id=foobar,--erase" 826 will translate to "west flash -- --board-id=foobar --erase" 827 """ 828 ) 829 parser.add_argument( 830 "--west-runner", 831 help="""Uses the specified west runner instead of default when running 832 with --west-flash. 833 834 E.g "twister --device-testing --device-serial /dev/ttyACM0 835 --west-flash --west-runner=pyocd" 836 will translate to "west flash --runner pyocd" 837 """ 838 ) 839 840 parser.add_argument( 841 "-X", "--fixture", action="append", default=[], 842 help="Specify a fixture that a board might support.") 843 844 parser.add_argument( 845 "-x", "--extra-args", action="append", default=[], 846 help="""Extra CMake cache entries to define when building test cases. 847 May be called multiple times. The key-value entries will be 848 prefixed with -D before being passed to CMake. 849 E.g 850 "twister -x=USE_CCACHE=0" 851 will translate to 852 "cmake -DUSE_CCACHE=0" 853 which will ultimately disable ccache. 854 """ 855 ) 856 857 parser.add_argument( 858 "-y", "--dry-run", action="store_true", 859 help="""Create the filtered list of test cases, but don't actually 860 run them. Useful if you're just interested in the test plan 861 generated for every run and saved in the specified output 862 directory (testplan.json). 863 """) 864 865 parser.add_argument("extra_test_args", nargs=argparse.REMAINDER, 866 help="Additional args following a '--' are passed to the test binary") 867 868 parser.add_argument("--alt-config-root", action="append", default=[], 869 help="Alternative test configuration root/s. When a test is found, " 870 "Twister will check if a test configuration file exist in any of " 871 "the alternative test configuration root folders. For example, " 872 "given $test_root/tests/foo/testcase.yaml, Twister will use " 873 "$alt_config_root/tests/foo/testcase.yaml if it exists") 874 875 return parser 876 877 878def parse_arguments( 879 parser: argparse.ArgumentParser, 880 args, 881 options = None, 882 on_init=True 883) -> argparse.Namespace: 884 if options is None: 885 options = parser.parse_args(args) 886 887 # Very early error handling 888 if options.short_build_path and not options.ninja: 889 logger.error("--short-build-path requires Ninja to be enabled") 890 sys.exit(1) 891 892 if options.device_serial_pty and os.name == "nt": # OS is Windows 893 logger.error("--device-serial-pty is not supported on Windows OS") 894 sys.exit(1) 895 896 if not options.testsuite_root: 897 # if we specify a test scenario which is part of a suite directly, do 898 # not set testsuite root to default, just point to the test directory 899 # directly. 900 if options.test: 901 for scenario in options.test: 902 if dirname := os.path.dirname(scenario): 903 options.testsuite_root.append(dirname) 904 905 # check again and make sure we have something set 906 if not options.testsuite_root: 907 options.testsuite_root = [os.path.join(ZEPHYR_BASE, "tests"), 908 os.path.join(ZEPHYR_BASE, "samples")] 909 910 if options.last_metrics or options.compare_report: 911 options.enable_size_report = True 912 913 if options.footprint_report: 914 options.create_rom_ram_report = True 915 916 if options.aggressive_no_clean: 917 options.no_clean = True 918 919 if options.coverage: 920 options.enable_coverage = True 921 922 if options.enable_coverage and not options.coverage_platform: 923 options.coverage_platform = options.platform 924 925 if ( 926 (not options.coverage) 927 and (options.disable_coverage_aggregation or options.coverage_per_instance) 928 ): 929 logger.error("Enable coverage reporting to set its aggregation mode.") 930 sys.exit(1) 931 932 if ( 933 options.coverage 934 and options.disable_coverage_aggregation and (not options.coverage_per_instance) 935 ): 936 logger.error("At least one coverage reporting mode should be enabled: " 937 "either aggregation, or per-instance, or both.") 938 sys.exit(1) 939 940 if options.coverage_formats: 941 for coverage_format in options.coverage_formats.split(','): 942 if coverage_format not in supported_coverage_formats[options.coverage_tool]: 943 logger.error(f"Unsupported coverage report formats:'{options.coverage_formats}' " 944 f"for {options.coverage_tool}") 945 sys.exit(1) 946 947 if options.enable_valgrind and not shutil.which("valgrind"): 948 logger.error("valgrind enabled but valgrind executable not found") 949 sys.exit(1) 950 951 if ( 952 (not options.device_testing) 953 and (options.device_serial or options.device_serial_pty or options.hardware_map) 954 ): 955 logger.error( 956 "Use --device-testing with --device-serial, or --device-serial-pty, or --hardware-map." 957 ) 958 sys.exit(1) 959 960 if ( 961 options.device_testing 962 and (options.device_serial or options.device_serial_pty) and len(options.platform) != 1 963 ): 964 logger.error("When --device-testing is used with --device-serial " 965 "or --device-serial-pty, exactly one platform must " 966 "be specified") 967 sys.exit(1) 968 969 if options.device_flash_with_test and not options.device_testing: 970 logger.error("--device-flash-with-test requires --device_testing") 971 sys.exit(1) 972 973 if options.flash_before and options.device_flash_with_test: 974 logger.error("--device-flash-with-test does not apply when --flash-before is used") 975 sys.exit(1) 976 977 if options.shuffle_tests and options.subset is None: 978 logger.error("--shuffle-tests requires --subset") 979 sys.exit(1) 980 981 if options.shuffle_tests_seed and options.shuffle_tests is None: 982 logger.error("--shuffle-tests-seed requires --shuffle-tests") 983 sys.exit(1) 984 985 if options.size: 986 from twisterlib.size_calc import SizeCalculator 987 for fn in options.size: 988 sc = SizeCalculator(fn, []) 989 sc.size_report() 990 sys.exit(0) 991 992 if options.footprint_from_buildlog: 993 logger.warning("WARNING: Using --footprint-from-buildlog will give inconsistent results " 994 "for configurations using sysbuild. It is recommended to not use this flag " 995 "when building configurations using sysbuild.") 996 if not options.enable_size_report: 997 logger.error("--footprint-from-buildlog requires --enable-size-report") 998 sys.exit(1) 999 1000 if len(options.extra_test_args) > 0: 1001 # extra_test_args is a list of CLI args that Twister did not recognize 1002 # and are intended to be passed through to the ztest executable. This 1003 # list should begin with a "--". If not, there is some extra 1004 # unrecognized arg(s) that shouldn't be there. Tell the user there is a 1005 # syntax error. 1006 if options.extra_test_args[0] != "--": 1007 try: 1008 double_dash = options.extra_test_args.index("--") 1009 except ValueError: 1010 double_dash = len(options.extra_test_args) 1011 unrecognized = " ".join(options.extra_test_args[0:double_dash]) 1012 1013 logger.error( 1014 f"Unrecognized arguments found: '{unrecognized}'." 1015 " Use -- to delineate extra arguments for test binary or pass -h for help." 1016 ) 1017 1018 sys.exit(1) 1019 1020 # Strip off the initial "--" following validation. 1021 options.extra_test_args = options.extra_test_args[1:] 1022 1023 if on_init and not options.allow_installed_plugin and PYTEST_PLUGIN_INSTALLED: 1024 logger.error("By default Twister should work without pytest-twister-harness " 1025 "plugin being installed, so please, uninstall it by " 1026 "`pip uninstall pytest-twister-harness` and `git clean " 1027 "-dxf scripts/pylib/pytest-twister-harness`.") 1028 sys.exit(1) 1029 elif on_init and options.allow_installed_plugin and PYTEST_PLUGIN_INSTALLED: 1030 logger.warning("You work with installed version of " 1031 "pytest-twister-harness plugin.") 1032 1033 return options 1034 1035def strip_ansi_sequences(s: str) -> str: 1036 """Remove ANSI escape sequences from a string.""" 1037 return re.sub(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', "", s) 1038 1039class TwisterEnv: 1040 1041 def __init__(self, options : argparse.Namespace, default_options=None) -> None: 1042 self.version = "Unknown" 1043 self.toolchain = None 1044 self.commit_date = "Unknown" 1045 self.run_date = None 1046 self.options = options 1047 self.default_options = default_options 1048 1049 if options.ninja: 1050 self.generator_cmd = "ninja" 1051 self.generator = "Ninja" 1052 else: 1053 self.generator_cmd = "make" 1054 self.generator = "Unix Makefiles" 1055 logger.info(f"Using {self.generator}..") 1056 1057 self.test_roots = options.testsuite_root 1058 1059 if not isinstance(options.board_root, list): 1060 self.board_roots = [options.board_root] 1061 else: 1062 self.board_roots = options.board_root 1063 self.outdir = os.path.abspath(options.outdir) 1064 1065 self.snippet_roots = [Path(ZEPHYR_BASE)] 1066 modules = zephyr_module.parse_modules(ZEPHYR_BASE) 1067 for module in modules: 1068 snippet_root = module.meta.get("build", {}).get("settings", {}).get("snippet_root") 1069 if snippet_root: 1070 self.snippet_roots.append(Path(module.project) / snippet_root) 1071 1072 1073 self.soc_roots = [Path(ZEPHYR_BASE), Path(ZEPHYR_BASE) / 'subsys' / 'testsuite'] 1074 self.dts_roots = [Path(ZEPHYR_BASE)] 1075 self.arch_roots = [Path(ZEPHYR_BASE)] 1076 1077 for module in modules: 1078 soc_root = module.meta.get("build", {}).get("settings", {}).get("soc_root") 1079 if soc_root: 1080 self.soc_roots.append(Path(module.project) / Path(soc_root)) 1081 dts_root = module.meta.get("build", {}).get("settings", {}).get("dts_root") 1082 if dts_root: 1083 self.dts_roots.append(Path(module.project) / Path(dts_root)) 1084 arch_root = module.meta.get("build", {}).get("settings", {}).get("arch_root") 1085 if arch_root: 1086 self.arch_roots.append(Path(module.project) / Path(arch_root)) 1087 1088 self.hwm = None 1089 1090 self.test_config = options.test_config 1091 1092 self.alt_config_root = options.alt_config_root 1093 1094 def non_default_options(self) -> dict: 1095 """Returns current command line options which are set to non-default values.""" 1096 diff = {} 1097 if not self.default_options: 1098 return diff 1099 dict_options = vars(self.options) 1100 dict_default = vars(self.default_options) 1101 for k in dict_options: 1102 if k not in dict_default or dict_options[k] != dict_default[k]: 1103 diff[k] = dict_options[k] 1104 return diff 1105 1106 def discover(self): 1107 self.check_zephyr_version() 1108 self.get_toolchain() 1109 self.run_date = datetime.now(timezone.utc).isoformat(timespec='seconds') 1110 1111 def check_zephyr_version(self): 1112 try: 1113 subproc = subprocess.run(["git", "describe", "--abbrev=12", "--always"], 1114 stdout=subprocess.PIPE, 1115 text=True, 1116 cwd=ZEPHYR_BASE) 1117 if subproc.returncode == 0: 1118 _version = subproc.stdout.strip() 1119 if _version: 1120 self.version = _version 1121 logger.info(f"Zephyr version: {self.version}") 1122 except OSError: 1123 logger.exception("Failure while reading Zephyr version.") 1124 1125 if self.version == "Unknown": 1126 logger.warning("Could not determine version") 1127 1128 try: 1129 subproc = subprocess.run(["git", "show", "-s", "--format=%cI", "HEAD"], 1130 stdout=subprocess.PIPE, 1131 text=True, 1132 cwd=ZEPHYR_BASE) 1133 if subproc.returncode == 0: 1134 self.commit_date = subproc.stdout.strip() 1135 except OSError: 1136 logger.exception("Failure while reading head commit date.") 1137 1138 @staticmethod 1139 def run_cmake_script(args=None): 1140 if args is None: 1141 args = [] 1142 script = os.fspath(args[0]) 1143 1144 logger.debug(f"Running cmake script {script}") 1145 1146 cmake_args = ["-D{}".format(a.replace('"', '')) for a in args[1:]] 1147 cmake_args.extend(['-P', script]) 1148 1149 cmake = shutil.which('cmake') 1150 if not cmake: 1151 msg = "Unable to find `cmake` in path" 1152 logger.error(msg) 1153 raise Exception(msg) 1154 cmd = [cmake] + cmake_args 1155 log_command(logger, "Calling cmake", cmd) 1156 1157 kwargs = dict() 1158 kwargs['stdout'] = subprocess.PIPE 1159 # CMake sends the output of message() to stderr unless it's STATUS 1160 kwargs['stderr'] = subprocess.STDOUT 1161 1162 p = subprocess.Popen(cmd, **kwargs) 1163 out, _ = p.communicate() 1164 1165 # It might happen that the environment adds ANSI escape codes like \x1b[0m, 1166 # for instance if twister is executed from inside a makefile. In such a 1167 # scenario it is then necessary to remove them, as otherwise the JSON decoding 1168 # will fail. 1169 out = strip_ansi_sequences(out.decode()) 1170 1171 if p.returncode == 0: 1172 msg = f"Finished running {args[0]}" 1173 logger.debug(msg) 1174 results = {"returncode": p.returncode, "msg": msg, "stdout": out} 1175 1176 else: 1177 logger.error(f"CMake script failure: {args[0]}") 1178 results = {"returncode": p.returncode, "returnmsg": out} 1179 1180 return results 1181 1182 def get_toolchain(self): 1183 toolchain_script = Path(ZEPHYR_BASE) / Path('cmake/verify-toolchain.cmake') 1184 result = self.run_cmake_script([toolchain_script, "FORMAT=json"]) 1185 1186 try: 1187 if result['returncode']: 1188 raise TwisterRuntimeError(f"E: {result['returnmsg']}") 1189 except Exception as e: 1190 print(str(e)) 1191 sys.exit(2) 1192 self.toolchain = json.loads(result['stdout'])['ZEPHYR_TOOLCHAIN_VARIANT'] 1193 logger.info(f"Using '{self.toolchain}' toolchain.") 1194