1#!/usr/bin/env python3
2# Copyright (c) 2023 Intel Corporation
3# Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved.
4#
5# SPDX-License-Identifier: Apache-2.0
6"""
7Tests for environment.py classes' methods
8"""
9
10from unittest import mock
11import os
12import pytest
13import shutil
14
15from contextlib import nullcontext
16
17import twisterlib.environment
18
19
20TESTDATA_1 = [
21    (
22        None,
23        None,
24        None,
25        ['--short-build-path', '-k'],
26        '--short-build-path requires Ninja to be enabled'
27    ),
28    (
29        'nt',
30        None,
31        None,
32        ['--device-serial-pty', 'dummy'],
33        '--device-serial-pty is not supported on Windows OS'
34    ),
35    (
36        None,
37        {
38            'exist': [],
39            'missing': ['valgrind']
40        },
41        None,
42        ['--enable-valgrind'],
43        'valgrind enabled but valgrind executable not found'
44    ),
45    (
46        None,
47        None,
48        None,
49        [
50            '--device-testing',
51            '--device-serial',
52            'dummy',
53        ],
54        'When --device-testing is used with --device-serial' \
55        ' or --device-serial-pty, exactly one platform must' \
56        ' be specified'
57    ),
58    (
59        None,
60        None,
61        None,
62        [
63            '--device-testing',
64            '--device-serial',
65            'dummy',
66            '--platform',
67            'dummy_platform1',
68            '--platform',
69            'dummy_platform2'
70        ],
71        'When --device-testing is used with --device-serial' \
72        ' or --device-serial-pty, exactly one platform must' \
73        ' be specified'
74    ),
75# Note the underscore.
76    (
77        None,
78        None,
79        None,
80        ['--device-flash-with-test'],
81        '--device-flash-with-test requires --device_testing'
82    ),
83    (
84        None,
85        None,
86        None,
87        ['--shuffle-tests'],
88        '--shuffle-tests requires --subset'
89    ),
90    (
91        None,
92        None,
93        None,
94        ['--shuffle-tests-seed', '0'],
95        '--shuffle-tests-seed requires --shuffle-tests'
96    ),
97    (
98        None,
99        None,
100        None,
101        ['/dummy/unrecognised/arg'],
102        'Unrecognized arguments found: \'/dummy/unrecognised/arg\'.' \
103        ' Use -- to delineate extra arguments for test binary' \
104        ' or pass -h for help.'
105    ),
106    (
107        None,
108        None,
109        True,
110        [],
111        'By default Twister should work without pytest-twister-harness' \
112        ' plugin being installed, so please, uninstall it by' \
113        ' `pip uninstall pytest-twister-harness` and' \
114        ' `git clean -dxf scripts/pylib/pytest-twister-harness`.'
115    ),
116]
117
118
119@pytest.mark.parametrize(
120    'os_name, which_dict, pytest_plugin, args, expected_error',
121    TESTDATA_1,
122    ids=[
123        'short build path without ninja',
124        'device-serial-pty on Windows',
125        'valgrind without executable',
126        'device serial without platform',
127        'device serial with multiple platforms',
128        'device flash with test without device testing',
129        'shuffle-tests without subset',
130        'shuffle-tests-seed without shuffle-tests',
131        'unrecognised argument',
132        'pytest-twister-harness installed'
133    ]
134)
135def test_parse_arguments_errors(
136    caplog,
137    os_name,
138    which_dict,
139    pytest_plugin,
140    args,
141    expected_error
142):
143    def mock_which(name):
144        if name in which_dict['missing']:
145            return False
146        elif name in which_dict['exist']:
147            return which_dict['path'][which_dict['exist']] \
148                if which_dict['path'][which_dict['exist']] \
149                else f'dummy/path/{name}'
150        else:
151            return f'dummy/path/{name}'
152
153    with mock.patch('sys.argv', ['twister'] + args):
154        parser = twisterlib.environment.add_parse_arguments()
155
156    if which_dict:
157        which_dict['path'] = {name: shutil.which(name) \
158            for name in which_dict['exist']}
159        which_mock = mock.Mock(side_effect=mock_which)
160
161    with mock.patch('os.name', os_name) \
162            if os_name is not None else nullcontext(), \
163         mock.patch('shutil.which', which_mock) \
164            if which_dict else nullcontext(), \
165         mock.patch('twisterlib.environment' \
166                    '.PYTEST_PLUGIN_INSTALLED', pytest_plugin) \
167            if pytest_plugin is not None else nullcontext():
168        with pytest.raises(SystemExit) as exit_info:
169            twisterlib.environment.parse_arguments(parser, args)
170
171    assert exit_info.value.code == 1
172    assert expected_error in ' '.join(caplog.text.split())
173
174
175def test_parse_arguments_errors_size():
176    """`options.size` is not an error, rather a different functionality."""
177
178    args = ['--size', 'dummy.elf']
179
180    with mock.patch('sys.argv', ['twister'] + args):
181        parser = twisterlib.environment.add_parse_arguments()
182
183    mock_calc_parent = mock.Mock()
184    mock_calc_parent.child = mock.Mock(return_value=mock.Mock())
185
186    def mock_calc(*args, **kwargs):
187        return mock_calc_parent.child(args, kwargs)
188
189    with mock.patch('twisterlib.size_calc.SizeCalculator', mock_calc):
190        with pytest.raises(SystemExit) as exit_info:
191            twisterlib.environment.parse_arguments(parser, args)
192
193    assert exit_info.value.code == 0
194
195    mock_calc_parent.child.assert_has_calls([mock.call(('dummy.elf', []), {})])
196    mock_calc_parent.child().size_report.assert_has_calls([mock.call()])
197
198
199def test_parse_arguments_warnings(caplog):
200    args = ['--allow-installed-plugin']
201
202    with mock.patch('sys.argv', ['twister'] + args):
203        parser = twisterlib.environment.add_parse_arguments()
204
205    with mock.patch('twisterlib.environment.PYTEST_PLUGIN_INSTALLED', True):
206        twisterlib.environment.parse_arguments(parser, args)
207
208    assert 'You work with installed version of' \
209           ' pytest-twister-harness plugin.' in ' '.join(caplog.text.split())
210
211
212TESTDATA_2 = [
213    (['--enable-size-report']),
214    (['--compare-report', 'dummy']),
215]
216
217
218@pytest.mark.parametrize(
219    'additional_args',
220    TESTDATA_2,
221    ids=['show footprint', 'compare report']
222)
223def test_parse_arguments(zephyr_base, additional_args):
224    args = ['--coverage', '--platform', 'dummy_platform'] + \
225           additional_args + ['--', 'dummy_extra_1', 'dummy_extra_2']
226
227    with mock.patch('sys.argv', ['twister'] + args):
228        parser = twisterlib.environment.add_parse_arguments()
229
230    options = twisterlib.environment.parse_arguments(parser, args)
231
232    assert os.path.join(zephyr_base, 'tests') in options.testsuite_root
233    assert os.path.join(zephyr_base, 'samples') in options.testsuite_root
234
235    assert options.enable_size_report
236
237    assert options.enable_coverage
238
239    assert options.coverage_platform == ['dummy_platform']
240
241    assert options.extra_test_args == ['dummy_extra_1', 'dummy_extra_2']
242
243
244TESTDATA_3 = [
245    (
246        mock.Mock(
247            ninja=True,
248            board_root=['dummy1', 'dummy2'],
249            testsuite_root=[
250                os.path.join('dummy', 'path', "tests"),
251                os.path.join('dummy', 'path', "samples")
252            ],
253            outdir='dummy_abspath',
254        ),
255        mock.Mock(
256            generator_cmd='ninja',
257            generator='Ninja',
258            test_roots=[
259                os.path.join('dummy', 'path', "tests"),
260                os.path.join('dummy', 'path', "samples")
261            ],
262            board_roots=['dummy1', 'dummy2'],
263            outdir='dummy_abspath',
264        )
265    ),
266    (
267        mock.Mock(
268            ninja=False,
269            board_root='dummy0',
270            testsuite_root=[
271                os.path.join('dummy', 'path', "tests"),
272                os.path.join('dummy', 'path', "samples")
273            ],
274            outdir='dummy_abspath',
275        ),
276        mock.Mock(
277            generator_cmd='make',
278            generator='Unix Makefiles',
279            test_roots=[
280                os.path.join('dummy', 'path', "tests"),
281                os.path.join('dummy', 'path', "samples")
282            ],
283            board_roots=['dummy0'],
284            outdir='dummy_abspath',
285        )
286    ),
287]
288
289
290@pytest.mark.parametrize(
291    'options, expected_env',
292    TESTDATA_3,
293    ids=[
294        'ninja',
295        'make'
296    ]
297)
298def test_twisterenv_init(options, expected_env):
299    original_abspath = os.path.abspath
300
301    def mocked_abspath(path):
302        if path == 'dummy_abspath':
303            return 'dummy_abspath'
304        elif isinstance(path, mock.Mock):
305            return None
306        else:
307            return original_abspath(path)
308
309    with mock.patch('os.path.abspath', side_effect=mocked_abspath):
310        twister_env = twisterlib.environment.TwisterEnv(options=options)
311
312    assert twister_env.generator_cmd == expected_env.generator_cmd
313    assert twister_env.generator == expected_env.generator
314
315    assert twister_env.test_roots == expected_env.test_roots
316
317    assert twister_env.board_roots == expected_env.board_roots
318    assert twister_env.outdir == expected_env.outdir
319
320
321def test_twisterenv_discover():
322    options = mock.Mock(
323        ninja=True
324    )
325
326    original_abspath = os.path.abspath
327
328    def mocked_abspath(path):
329        if path == 'dummy_abspath':
330            return 'dummy_abspath'
331        elif isinstance(path, mock.Mock):
332            return None
333        else:
334            return original_abspath(path)
335
336    with mock.patch('os.path.abspath', side_effect=mocked_abspath):
337        twister_env = twisterlib.environment.TwisterEnv(options=options)
338
339    mock_datetime = mock.Mock(
340        now=mock.Mock(
341            return_value=mock.Mock(
342                isoformat=mock.Mock(return_value='dummy_time')
343            )
344        )
345    )
346
347    with mock.patch.object(
348            twisterlib.environment.TwisterEnv,
349            'check_zephyr_version',
350            mock.Mock()) as mock_czv, \
351         mock.patch.object(
352            twisterlib.environment.TwisterEnv,
353            'get_toolchain',
354            mock.Mock()) as mock_gt, \
355         mock.patch('twisterlib.environment.datetime', mock_datetime):
356        twister_env.discover()
357
358    mock_czv.assert_called_once()
359    mock_gt.assert_called_once()
360    assert twister_env.run_date == 'dummy_time'
361
362
363TESTDATA_4 = [
364    (
365        mock.Mock(returncode=0, stdout='dummy stdout version'),
366        mock.Mock(returncode=0, stdout='dummy stdout date'),
367        ['Zephyr version: dummy stdout version'],
368        'dummy stdout version',
369        'dummy stdout date'
370    ),
371    (
372        mock.Mock(returncode=0, stdout=''),
373        mock.Mock(returncode=0, stdout='dummy stdout date'),
374        ['Could not determine version'],
375        'Unknown',
376        'dummy stdout date'
377    ),
378    (
379        mock.Mock(returncode=1, stdout='dummy stdout version'),
380        mock.Mock(returncode=0, stdout='dummy stdout date'),
381        ['Could not determine version'],
382        'Unknown',
383        'dummy stdout date'
384    ),
385    (
386        OSError,
387        mock.Mock(returncode=1),
388        ['Could not determine version'],
389        'Unknown',
390        'Unknown'
391    ),
392]
393
394
395@pytest.mark.parametrize(
396    'git_describe_return, git_show_return, expected_logs,' \
397    ' expected_version, expected_commit_date',
398    TESTDATA_4,
399    ids=[
400        'valid',
401        'no zephyr version on describe',
402        'error on git describe',
403        'execution error on git describe',
404    ]
405)
406def test_twisterenv_check_zephyr_version(
407    caplog,
408    git_describe_return,
409    git_show_return,
410    expected_logs,
411    expected_version,
412    expected_commit_date
413):
414    def mock_run(command, *args, **kwargs):
415        if all([keyword in command for keyword in ['git', 'describe']]):
416            if isinstance(git_describe_return, type) and \
417               issubclass(git_describe_return, Exception):
418                raise git_describe_return()
419            return git_describe_return
420        if all([keyword in command for keyword in ['git', 'show']]):
421            if isinstance(git_show_return, type) and \
422               issubclass(git_show_return, Exception):
423                raise git_show_return()
424            return git_show_return
425
426    options = mock.Mock(
427        ninja=True
428    )
429
430    original_abspath = os.path.abspath
431
432    def mocked_abspath(path):
433        if path == 'dummy_abspath':
434            return 'dummy_abspath'
435        elif isinstance(path, mock.Mock):
436            return None
437        else:
438            return original_abspath(path)
439
440    with mock.patch('os.path.abspath', side_effect=mocked_abspath):
441        twister_env = twisterlib.environment.TwisterEnv(options=options)
442
443    with mock.patch('subprocess.run', mock.Mock(side_effect=mock_run)):
444        twister_env.check_zephyr_version()
445    print(expected_logs)
446    print(caplog.text)
447    assert twister_env.version == expected_version
448    assert twister_env.commit_date == expected_commit_date
449    assert all([expected_log in caplog.text for expected_log in expected_logs])
450
451
452TESTDATA_5 = [
453    (
454        False,
455        None,
456        None,
457        'Unable to find `cmake` in path',
458        None
459    ),
460    (
461        True,
462        0,
463        b'somedummy\x1B[123-@d1770',
464        'Finished running dummy/script/path',
465        {
466            'returncode': 0,
467            'msg': 'Finished running dummy/script/path',
468            'stdout': 'somedummyd1770',
469        }
470    ),
471    (
472        True,
473        1,
474        b'another\x1B_dummy',
475        'CMake script failure: dummy/script/path',
476        {
477            'returncode': 1,
478            'returnmsg': 'anotherdummy'
479        }
480    ),
481]
482
483
484@pytest.mark.parametrize(
485    'find_cmake, return_code, out, expected_log, expected_result',
486    TESTDATA_5,
487    ids=[
488        'cmake not found',
489        'regex sanitation 1',
490        'regex sanitation 2'
491    ]
492)
493def test_twisterenv_run_cmake_script(
494    caplog,
495    find_cmake,
496    return_code,
497    out,
498    expected_log,
499    expected_result
500):
501    def mock_which(name, *args, **kwargs):
502        return 'dummy/cmake/path' if find_cmake else None
503
504    def mock_popen(command, *args, **kwargs):
505        return mock.Mock(
506            pid=0,
507            returncode=return_code,
508            communicate=mock.Mock(
509                return_value=(out, '')
510            )
511        )
512
513    args = ['dummy/script/path', 'var1=val1']
514
515    with mock.patch('shutil.which', mock_which), \
516         mock.patch('subprocess.Popen', mock.Mock(side_effect=mock_popen)), \
517         pytest.raises(Exception) \
518            if not find_cmake else nullcontext() as exception:
519        results = twisterlib.environment.TwisterEnv.run_cmake_script(args)
520
521    assert 'Running cmake script dummy/script/path' in caplog.text
522
523    assert expected_log in caplog.text
524
525    if exception is not None:
526        return
527
528    assert expected_result.items() <= results.items()
529
530
531TESTDATA_6 = [
532    (
533        {
534            'returncode': 0,
535            'stdout': '{\"ZEPHYR_TOOLCHAIN_VARIANT\": \"dummy toolchain\"}'
536        },
537        None,
538        'Using \'dummy toolchain\' toolchain.'
539    ),
540    (
541        {'returncode': 1},
542        2,
543        None
544    ),
545]
546
547
548@pytest.mark.parametrize(
549    'script_result, exit_value, expected_log',
550    TESTDATA_6,
551    ids=['valid', 'error']
552)
553def test_get_toolchain(caplog, script_result, exit_value, expected_log):
554    options = mock.Mock(
555        ninja=True
556    )
557
558    original_abspath = os.path.abspath
559
560    def mocked_abspath(path):
561        if path == 'dummy_abspath':
562            return 'dummy_abspath'
563        elif isinstance(path, mock.Mock):
564            return None
565        else:
566            return original_abspath(path)
567
568    with mock.patch('os.path.abspath', side_effect=mocked_abspath):
569        twister_env = twisterlib.environment.TwisterEnv(options=options)
570
571    with mock.patch.object(
572            twisterlib.environment.TwisterEnv,
573            'run_cmake_script',
574            mock.Mock(return_value=script_result)), \
575         pytest.raises(SystemExit) \
576            if exit_value is not None else nullcontext() as exit_info:
577        twister_env.get_toolchain()
578
579    if exit_info is not None:
580        assert exit_info.value.code == exit_value
581    else:
582        assert expected_log in caplog.text
583