1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions concerning addons to normal functions
7"""
8
9import importlib
10from unittest import mock
11import os
12import pkg_resources
13import pytest
14import re
15import shutil
16import subprocess
17import sys
18
19# pylint: disable=no-name-in-module
20from conftest import ZEPHYR_BASE, TEST_DATA, sample_filename_mock, testsuite_filename_mock
21from twisterlib.testplan import TestPlan
22
23
24class TestAddon:
25    @classmethod
26    def setup_class(cls):
27        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
28        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
29        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
30        cls.twister_module = importlib.util.module_from_spec(cls.spec)
31
32    @classmethod
33    def teardown_class(cls):
34        pass
35
36    @pytest.mark.parametrize(
37        'ubsan_flags, expected_exit_value',
38        [
39            # No sanitiser, no problem
40            ([], '0'),
41            # Sanitiser catches a mistake, error is raised
42            (['--enable-ubsan'], '1')
43        ],
44        ids=['no sanitiser', 'ubsan']
45    )
46    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
47    def test_enable_ubsan(self, out_path, ubsan_flags, expected_exit_value):
48        test_platforms = ['native_sim']
49        test_path = os.path.join(TEST_DATA, 'tests', 'san', 'ubsan')
50        args = ['-i', '--outdir', out_path, '-T', test_path] + \
51               ubsan_flags + \
52               [] + \
53               [val for pair in zip(
54                   ['-p'] * len(test_platforms), test_platforms
55               ) for val in pair]
56
57        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
58                pytest.raises(SystemExit) as sys_exit:
59            self.loader.exec_module(self.twister_module)
60
61        assert str(sys_exit.value) == expected_exit_value
62
63    @pytest.mark.parametrize(
64        'lsan_flags, expected_exit_value',
65        [
66            # No sanitiser, no problem
67            ([], '0'),
68            # Sanitiser catches a mistake, error is raised
69            (['--enable-asan', '--enable-lsan'], '1')
70        ],
71        ids=['no sanitiser', 'lsan']
72    )
73    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
74    def test_enable_lsan(self, out_path, lsan_flags, expected_exit_value):
75        test_platforms = ['native_sim']
76        test_path = os.path.join(TEST_DATA, 'tests', 'san', 'lsan')
77        args = ['-i', '--outdir', out_path, '-T', test_path] + \
78               lsan_flags + \
79               [] + \
80               [val for pair in zip(
81                   ['-p'] * len(test_platforms), test_platforms
82               ) for val in pair]
83
84        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
85                pytest.raises(SystemExit) as sys_exit:
86            self.loader.exec_module(self.twister_module)
87
88        assert str(sys_exit.value) == expected_exit_value
89
90    @pytest.mark.parametrize(
91        'asan_flags, expected_exit_value, expect_asan',
92        [
93            # No sanitiser, no problem
94            # Note that on some runs it may fail,
95            # as the process is killed instead of ending normally.
96            # This is not 100% repeatable, so this test is removed for now.
97            # ([], '0', False),
98            # Sanitiser catches a mistake, error is raised
99            (['--enable-asan'], '1', True)
100        ],
101        ids=[
102            #'no sanitiser',
103            'asan'
104        ]
105    )
106    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
107    def test_enable_asan(self, capfd, out_path, asan_flags, expected_exit_value, expect_asan):
108        test_platforms = ['native_sim']
109        test_path = os.path.join(TEST_DATA, 'tests', 'san', 'asan')
110        args = ['-i', '-W', '--outdir', out_path, '-T', test_path] + \
111               asan_flags + \
112               [] + \
113               [val for pair in zip(
114                   ['-p'] * len(test_platforms), test_platforms
115               ) for val in pair]
116
117        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
118                pytest.raises(SystemExit) as sys_exit:
119            self.loader.exec_module(self.twister_module)
120
121        assert str(sys_exit.value) == expected_exit_value
122
123        out, err = capfd.readouterr()
124        sys.stdout.write(out)
125        sys.stderr.write(err)
126
127        asan_template = r'^==\d+==ERROR:\s+AddressSanitizer:'
128        assert expect_asan == bool(re.search(asan_template, err, re.MULTILINE))
129
130    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
131    def test_extra_test_args(self, capfd, out_path):
132        test_platforms = ['native_sim']
133        test_path = os.path.join(TEST_DATA, 'tests', 'params', 'dummy')
134        args = ['-i', '--outdir', out_path, '-T', test_path] + \
135               [] + \
136               ['-vvv'] + \
137               [val for pair in zip(
138                   ['-p'] * len(test_platforms), test_platforms
139               ) for val in pair] + \
140               ['--', '-list']
141
142        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
143                pytest.raises(SystemExit) as sys_exit:
144            self.loader.exec_module(self.twister_module)
145
146        # Use of -list makes tests not run.
147        # Thus, the tests 'failed'.
148        assert str(sys_exit.value) == '1'
149
150        out, err = capfd.readouterr()
151        sys.stdout.write(out)
152        sys.stderr.write(err)
153
154        expected_test_names = [
155            'param_tests::test_assert1',
156            'param_tests::test_assert2',
157            'param_tests::test_assert3',
158        ]
159        assert all([testname in err for testname in expected_test_names])
160
161    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
162    def test_extra_args(self, caplog, out_path):
163        test_platforms = ['qemu_x86', 'intel_adl_crb']
164        path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2')
165        args = ['--outdir', out_path, '-T', path] + \
166               ['--extra-args', 'USE_CCACHE=0', '--extra-args', 'DUMMY=1'] + \
167               [] + \
168               [val for pair in zip(
169                   ['-p'] * len(test_platforms), test_platforms
170               ) for val in pair]
171
172        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
173                pytest.raises(SystemExit) as sys_exit:
174            self.loader.exec_module(self.twister_module)
175
176        assert str(sys_exit.value) == '0'
177
178        with open(os.path.join(out_path, 'twister.log')) as f:
179            twister_log = f.read()
180
181        pattern_cache = r'Calling cmake: [^\n]+ -DUSE_CCACHE=0 [^\n]+\n'
182        pattern_dummy = r'Calling cmake: [^\n]+ -DDUMMY=1 [^\n]+\n'
183
184        assert ' -DUSE_CCACHE=0 ' in twister_log
185        res = re.search(pattern_cache, twister_log)
186        assert res
187
188        assert ' -DDUMMY=1 ' in twister_log
189        res = re.search(pattern_dummy, twister_log)
190        assert res
191
192    # This test is not side-effect free.
193    # It installs and uninstalls pytest-twister-harness using pip
194    # It uses pip to check whether that plugin is previously installed
195    # and reinstalls it if detected at the start of its run.
196    # However, it does NOT restore the original plugin, ONLY reinstalls it.
197    @pytest.mark.parametrize(
198        'allow_flags, do_install, expected_exit_value, expected_logs',
199        [
200            ([], True, '1', ['By default Twister should work without pytest-twister-harness'
201                             ' plugin being installed, so please, uninstall it by'
202                             ' `pip uninstall pytest-twister-harness` and'
203                             ' `git clean -dxf scripts/pylib/pytest-twister-harness`.']),
204            (['--allow-installed-plugin'], True, '0', ['You work with installed version'
205                                                       ' of pytest-twister-harness plugin.']),
206            ([], False, '0', []),
207            (['--allow-installed-plugin'], False, '0', []),
208        ],
209        ids=['installed, but not allowed', 'installed, allowed',
210             'not installed, not allowed', 'not installed, but allowed']
211    )
212    @mock.patch.object(TestPlan, 'SAMPLE_FILENAME', sample_filename_mock)
213    def test_allow_installed_plugin(self, caplog, out_path, allow_flags, do_install,
214                                    expected_exit_value, expected_logs):
215        environment_twister_module = importlib.import_module('twisterlib.environment')
216        harness_twister_module = importlib.import_module('twisterlib.harness')
217        runner_twister_module = importlib.import_module('twisterlib.runner')
218
219        pth_path = os.path.join(ZEPHYR_BASE, 'scripts', 'pylib', 'pytest-twister-harness')
220        check_installed_command = [sys.executable, '-m', 'pip', 'list']
221        install_command = [sys.executable, '-m', 'pip', 'install', '--no-input', pth_path]
222        uninstall_command = [sys.executable, '-m', 'pip', 'uninstall', '--yes',
223                             'pytest-twister-harness']
224
225        def big_uninstall():
226            pth_path = os.path.join(ZEPHYR_BASE, 'scripts', 'pylib', 'pytest-twister-harness')
227
228            subprocess.run(uninstall_command, check=True,)
229
230            # For our registration to work, we have to delete the installation cache
231            additional_cache_paths = [
232                # Plugin cache
233                os.path.join(pth_path, 'src', 'pytest_twister_harness.egg-info'),
234                # Additional caches
235                os.path.join(pth_path, 'src', 'pytest_twister_harness', '__pycache__'),
236                os.path.join(pth_path, 'src', 'pytest_twister_harness', 'device', '__pycache__'),
237                os.path.join(pth_path, 'src', 'pytest_twister_harness', 'helpers', '__pycache__'),
238                os.path.join(pth_path, 'src', 'pytest_twister_harness', 'build'),
239            ]
240
241            for additional_cache_path in additional_cache_paths:
242                if os.path.exists(additional_cache_path):
243                    if os.path.isfile(additional_cache_path):
244                        os.unlink(additional_cache_path)
245                    else:
246                        shutil.rmtree(additional_cache_path)
247
248        # To refresh the PYTEST_PLUGIN_INSTALLED global variable
249        def refresh_plugin_installed_variable():
250            pkg_resources._initialize_master_working_set()
251            importlib.reload(environment_twister_module)
252            importlib.reload(harness_twister_module)
253            importlib.reload(runner_twister_module)
254
255        check_installed_result = subprocess.run(check_installed_command, check=True,
256                                                capture_output=True, text=True)
257        previously_installed = 'pytest-twister-harness' in check_installed_result.stdout
258
259        # To ensure consistent test start
260        big_uninstall()
261
262        if do_install:
263            subprocess.run(install_command, check=True)
264
265        # Refresh before the test, no matter the testcase
266        refresh_plugin_installed_variable()
267
268        test_platforms = ['native_sim']
269        test_path = os.path.join(TEST_DATA, 'samples', 'pytest', 'shell')
270        args = ['-i', '--outdir', out_path, '-T', test_path] + \
271               allow_flags + \
272               [] + \
273               [val for pair in zip(
274                   ['-p'] * len(test_platforms), test_platforms
275               ) for val in pair]
276
277        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
278                pytest.raises(SystemExit) as sys_exit:
279            self.loader.exec_module(self.twister_module)
280
281        # To ensure consistent test exit, prevent dehermetisation
282        if do_install:
283            big_uninstall()
284
285        # To restore previously-installed plugin as well as we can
286        if previously_installed:
287            subprocess.run(install_command, check=True)
288
289        if previously_installed or do_install:
290            refresh_plugin_installed_variable()
291
292        assert str(sys_exit.value) == expected_exit_value
293
294        assert all([log in caplog.text for log in expected_logs])
295
296    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
297    def test_pytest_args(self, out_path):
298        test_platforms = ['native_sim']
299        test_path = os.path.join(TEST_DATA, 'tests', 'pytest')
300        args = ['-i', '--outdir', out_path, '-T', test_path] + \
301               ['--pytest-args=--custom-pytest-arg', '--pytest-args=foo',
302                '--pytest-args=--cmdopt', '--pytest-args=.'] + \
303               [] + \
304               [val for pair in zip(
305                   ['-p'] * len(test_platforms), test_platforms
306               ) for val in pair]
307
308        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
309                pytest.raises(SystemExit) as sys_exit:
310            self.loader.exec_module(self.twister_module)
311
312        # YAML was modified so that the test will fail without command line override.
313        assert str(sys_exit.value) == '0'
314
315    @pytest.mark.parametrize(
316        'valgrind_flags, expected_exit_value',
317        [
318            # No sanitiser, leak is ignored
319            ([], '0'),
320            # Sanitiser catches a mistake, error is raised
321            (['--enable-valgrind'], '1')
322        ],
323        ids=['no valgrind', 'valgrind']
324    )
325    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
326    def test_enable_valgrind(self, capfd, out_path, valgrind_flags, expected_exit_value):
327        test_platforms = ['native_sim']
328        test_path = os.path.join(TEST_DATA, 'tests', 'san', 'val')
329        args = ['-i', '--outdir', out_path, '-T', test_path] + \
330               valgrind_flags + \
331               [] + \
332               [val for pair in zip(
333                   ['-p'] * len(test_platforms), test_platforms
334               ) for val in pair]
335
336        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
337                pytest.raises(SystemExit) as sys_exit:
338            self.loader.exec_module(self.twister_module)
339
340        assert str(sys_exit.value) == expected_exit_value
341