1# SPDX-License-Identifier: GPL-2.0
2# Copyright 2024 Google LLC
3
4import pytest
5import re
6
7# List of test suites we expect to find with 'ut info' and 'ut all'
8EXPECTED_SUITES = [
9    'addrmap', 'bdinfo', 'bloblist', 'bootm', 'bootstd',
10    'cmd', 'common', 'dm', 'env', 'exit', 'fdt_overlay',
11    'fdt', 'font', 'hush', 'lib',
12    'loadm', 'log', 'mbr', 'measurement', 'mem',
13    'pci_mps', 'setexpr', 'upl',
14    ]
15
16
17# Set this to True to aid debugging of tests
18DEBUG_ME = False
19
20
21def collect_info(ubman, output):
22    """Process the output from 'ut all'
23
24    Args:
25        ubman: U-Boot console object
26        output: Output from running 'ut all'
27
28    Returns:
29        tuple:
30            set: suite names that were found in output
31            set: test names that were found in output
32            dict: test count for each suite:
33                key: suite name
34                value: number of tests for the suite found in output
35            set: missing suites (compared to EXPECTED_SUITES)
36            set: extra suites (compared to EXPECTED_SUITES)
37    """
38    suites = set()
39    tests = set()
40    cur_suite = None
41    test_count = None
42    exp_test_count = {}
43
44    # Collect suites{}
45    for line in output.splitlines():
46        line = line.rstrip()
47        if DEBUG_ME:
48            ubman.log.info(f'line: {line}')
49        m = re.search('----Running ([^ ]*) tests----', line)
50        if m:
51            if DEBUG_ME and cur_suite and cur_suite != 'info':
52                ubman.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}')
53
54            cur_suite = m.group(1)
55            if DEBUG_ME:
56                ubman.log.info(f'cur_suite: {cur_suite}')
57            suites.add(cur_suite)
58
59            test_count = 0
60        m = re.match(rf'Running (\d+) {cur_suite} tests', line)
61        if m:
62            exp_test_count[cur_suite] = int(m.group(1))
63        m = re.search(r'Test: (\w*): ([-a-z0-9_]*\.c)?( .*)?', line)
64        if m:
65            test_name = m.group(1)
66            msg = m.group(3)
67            if DEBUG_ME:
68                ubman.log.info(f"test_name {test_name} msg '{msg}'")
69            full_name = f'{cur_suite}.{test_name}'
70            if msg == ' (flat tree)' and full_name not in tests:
71                tests.add(full_name)
72                test_count += 1
73            if not msg or 'skipped as it is manual' in msg:
74                tests.add(full_name)
75                test_count += 1
76        if DEBUG_ME:
77            ubman.log.info(f'test_count {test_count}')
78    if DEBUG_ME:
79        ubman.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}')
80        ubman.log.info(f"Tests: {' '.join(sorted(list(tests)))}")
81
82    # Figure out what is missing, or extra
83    missing = set()
84    extra = set(suites)
85    for suite in EXPECTED_SUITES:
86        if suite in extra:
87            extra.remove(suite)
88        else:
89            missing.add(suite)
90
91    return suites, tests, exp_test_count, missing, extra
92
93
94def process_ut_info(ubman, output):
95    """Process the output of the 'ut info' command
96
97    Args:
98        ubman: U-Boot console object
99        output: Output from running 'ut all'
100
101    Returns:
102        tuple:
103            int: Number of suites reported
104            int: Number of tests reported
105            dict: test count for each suite:
106                key: suite name
107                value: number of tests reported for the suite
108
109    """
110    suite_count = None
111    total_test_count = None
112    test_count = {}
113    for line in output.splitlines():
114        line = line.rstrip()
115        if DEBUG_ME:
116            ubman.log.info(f'line: {line}')
117        m = re.match(r'Test suites: (.*)', line)
118        if m:
119            suite_count = int(m.group(1))
120        m = re.match(r'Total tests: (.*)', line)
121        if m:
122            total_test_count = int(m.group(1))
123        m = re.match(r'  *([0-9?]*)  (\w*)', line)
124        if m:
125            test_count[m.group(2)] = m.group(1)
126    return suite_count, total_test_count, test_count
127
128
129@pytest.mark.buildconfigspec('sandbox')
130@pytest.mark.notbuildconfigspec('sandbox_spl')
131@pytest.mark.notbuildconfigspec('sandbox64')
132# This test is disabled since it fails; remove the leading 'x' to try it
133def xtest_suite(ubman, u_boot_config):
134    """Perform various checks on the unit tests, including:
135
136       - The number of suites matches that reported by the 'ut info'
137       - Where available, the number of tests is each suite matches that
138         reported by 'ut -s info'
139       - The total number of tests adds up to the total that are actually run
140         with 'ut all'
141       - All suites are run with 'ut all'
142       - The expected set of suites is run (the list is hard-coded in this test)
143
144    """
145    buildconfig = u_boot_config.buildconfig
146    with ubman.log.section('Run all unit tests'):
147        # ut hush hush_test_simple_dollar prints "Unknown command" on purpose.
148        with ubman.disable_check('unknown_command'):
149            output = ubman.run_command('ut all')
150
151    # Process the output from the run
152    with ubman.log.section('Check output'):
153        suites, all_tests, exp_test_count, missing, extra = collect_info(ubman,
154                                                                         output)
155    ubman.log.info(f'missing {missing}')
156    ubman.log.info(f'extra {extra}')
157
158    # Make sure we got a test count for each suite
159    assert not (suites - exp_test_count.keys())
160
161    # Deal with missing suites
162    with ubman.log.section('Check missing suites'):
163        if 'config_cmd_seama' not in buildconfig:
164            ubman.log.info("CMD_SEAMA not enabled: Ignoring suite 'seama'")
165            missing.discard('seama')
166
167    # Run 'ut info' and compare with the log results
168    with ubman.log.section('Check suite test-counts'):
169        output = ubman.run_command('ut -s info')
170
171        suite_count, total_test_count, test_count = process_ut_info(ubman,
172                                                                    output)
173
174        if missing or extra:
175            ubman.log.info(f"suites: {' '.join(sorted(list(suites)))}")
176            ubman.log.error(f'missing: {sorted(list(missing))}')
177            ubman.log.error(f'extra: {sorted(list(extra))}')
178
179        assert not missing, f'Missing suites {missing}'
180        assert not extra, f'Extra suites {extra}'
181
182        ubman.log.info(str(exp_test_count))
183        for suite in EXPECTED_SUITES:
184            assert test_count[suite] in ['?', str(exp_test_count[suite])], \
185                f'suite {suite} expected {exp_test_count[suite]}'
186
187        assert suite_count == len(EXPECTED_SUITES)
188        assert total_test_count == len(all_tests)
189
190    # Run three suites
191    with ubman.log.section('Check multiple suites'):
192        output = ubman.run_command('ut bloblist,setexpr,mem')
193        assert 'Suites run: 3' in output
194
195    # Run a particular test
196    with ubman.log.section('Check single test'):
197        output = ubman.run_command('ut bloblist reloc')
198        assert 'Test: reloc: bloblist.c' in output
199
200    # Run tests multiple times
201    with ubman.log.section('Check multiple runs'):
202        output = ubman.run_command('ut -r2 bloblist')
203        lines = output.splitlines()
204        run = len([line for line in lines if 'Test:' in line])
205        count = re.search(r'Tests run: (\d*)', lines[-1]).group(1)
206
207        assert run == 2 * int(count)
208