1#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions related to the quarantine.
7"""
8
9import importlib
10from unittest import mock
11import os
12import pytest
13import re
14import sys
15import json
16
17# pylint: disable=duplicate-code
18# pylint: disable=no-name-in-module
19from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
20from twisterlib.testplan import TestPlan
21
22
23class TestQuarantine:
24    @classmethod
25    def setup_class(cls):
26        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
27        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
28        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
29        cls.twister_module = importlib.util.module_from_spec(cls.spec)
30
31    @classmethod
32    def teardown_class(cls):
33        pass
34
35    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
36    def test_quarantine_verify(self, out_path):
37        test_platforms = ['qemu_x86', 'intel_adl_crb']
38        path = os.path.join(TEST_DATA, 'tests', 'dummy')
39        quarantine_path = os.path.join(TEST_DATA, 'twister-quarantine-list.yml')
40        args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \
41               ['--quarantine-verify'] + \
42               ['--quarantine-list', quarantine_path] + \
43               [val for pair in zip(
44                   ['-p'] * len(test_platforms), test_platforms
45               ) for val in pair]
46
47        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
48                pytest.raises(SystemExit) as sys_exit:
49            self.loader.exec_module(self.twister_module)
50
51        with open(os.path.join(out_path, 'testplan.json')) as f:
52            j = json.load(f)
53
54        # Quarantine-verify "swaps" statuses. The ones that are in quarantine list
55        # should no longer be quarantined, and the ones that are not in the list
56        # should be quarantined. Remove "quarantined" tests from "verify" testplan
57        # to count what should be verified.
58        filtered_j = [
59           (ts['platform'], ts['name']) \
60               for ts in j['testsuites'] \
61               if ts['status'] != "skipped"
62        ]
63
64        assert str(sys_exit.value) == '0'
65
66        assert len(filtered_j) == 2
67
68    @pytest.mark.parametrize(
69        'test_path, test_platforms, quarantine_directory',
70        [
71            (
72                os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
73                ['qemu_x86', 'qemu_x86_64', 'intel_adl_crb'],
74                os.path.join(TEST_DATA, 'twister-quarantine-list.yml'),
75            ),
76        ],
77        ids=[
78            'quarantine',
79        ],
80    )
81    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
82    def test_quarantine_list(self, capfd, out_path, test_path, test_platforms, quarantine_directory):
83        args = ['--outdir', out_path, '-T', test_path] +\
84               ['--quarantine-list', quarantine_directory] + \
85               ['-vv', '-ll', 'DEBUG'] + \
86               [val for pair in zip(
87                   ['-p'] * len(test_platforms), test_platforms
88               ) for val in pair]
89
90        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
91            pytest.raises(SystemExit) as sys_exit:
92            self.loader.exec_module(self.twister_module)
93
94        out, err = capfd.readouterr()
95        sys.stdout.write(out)
96        sys.stderr.write(err)
97
98        board1_match1 = re.search('agnostic/group2/dummy.agnostic.group2 SKIPPED: Quarantine: test '
99                               'intel_adl_crb', err)
100        board1_match2 = re.search(
101            'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 SKIPPED: Quarantine: test '
102            'intel_adl_crb',
103            err)
104        qemu_64_match = re.search(
105            'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 SKIPPED: Quarantine: test '
106            'qemu_x86_64',
107            err)
108        all_platforms_match = re.search(
109            'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test '
110            'all platforms',
111            err)
112        all_platforms_match2 = re.search(
113            'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test '
114            'all platforms',
115            err)
116        all_platforms_match3 = re.search(
117            'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test '
118            'all platforms',
119            err)
120
121        assert board1_match1 and board1_match2, 'platform quarantine not working properly'
122        assert qemu_64_match, 'platform quarantine on scenario not working properly'
123        assert all_platforms_match and all_platforms_match2 and all_platforms_match3, 'scenario ' \
124                                                                                      'quarantine' \
125                                                                                      ' not work ' \
126                                                                                      'properly'
127
128        assert str(sys_exit.value) == '0'
129