1#!/usr/bin/env python3
2"""Describe the test coverage of PSA functions in terms of return statuses.
3
41. Build Mbed Crypto with -DRECORD_PSA_STATUS_COVERAGE_LOG
52. Run psa_collect_statuses.py
6
7The output is a series of line of the form "psa_foo PSA_ERROR_XXX". Each
8function/status combination appears only once.
9
10This script must be run from the top of an Mbed Crypto source tree.
11The build command is "make -DRECORD_PSA_STATUS_COVERAGE_LOG", which is
12only supported with make (as opposed to CMake or other build methods).
13"""
14
15# Copyright The Mbed TLS Contributors
16# SPDX-License-Identifier: Apache-2.0
17#
18# Licensed under the Apache License, Version 2.0 (the "License"); you may
19# not use this file except in compliance with the License.
20# You may obtain a copy of the License at
21#
22# http://www.apache.org/licenses/LICENSE-2.0
23#
24# Unless required by applicable law or agreed to in writing, software
25# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
26# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
27# See the License for the specific language governing permissions and
28# limitations under the License.
29
30import argparse
31import os
32import subprocess
33import sys
34
35DEFAULT_STATUS_LOG_FILE = 'tests/statuses.log'
36DEFAULT_PSA_CONSTANT_NAMES = 'programs/psa/psa_constant_names'
37
38class Statuses:
39    """Information about observed return statues of API functions."""
40
41    def __init__(self):
42        self.functions = {}
43        self.codes = set()
44        self.status_names = {}
45
46    def collect_log(self, log_file_name):
47        """Read logs from RECORD_PSA_STATUS_COVERAGE_LOG.
48
49        Read logs produced by running Mbed Crypto test suites built with
50        -DRECORD_PSA_STATUS_COVERAGE_LOG.
51        """
52        with open(log_file_name) as log:
53            for line in log:
54                value, function, tail = line.split(':', 2)
55                if function not in self.functions:
56                    self.functions[function] = {}
57                fdata = self.functions[function]
58                if value not in self.functions[function]:
59                    fdata[value] = []
60                fdata[value].append(tail)
61                self.codes.add(int(value))
62
63    def get_constant_names(self, psa_constant_names):
64        """Run psa_constant_names to obtain names for observed numerical values."""
65        values = [str(value) for value in self.codes]
66        cmd = [psa_constant_names, 'status'] + values
67        output = subprocess.check_output(cmd).decode('ascii')
68        for value, name in zip(values, output.rstrip().split('\n')):
69            self.status_names[value] = name
70
71    def report(self):
72        """Report observed return values for each function.
73
74        The report is a series of line of the form "psa_foo PSA_ERROR_XXX".
75        """
76        for function in sorted(self.functions.keys()):
77            fdata = self.functions[function]
78            names = [self.status_names[value] for value in fdata.keys()]
79            for name in sorted(names):
80                sys.stdout.write('{} {}\n'.format(function, name))
81
82def collect_status_logs(options):
83    """Build and run unit tests and report observed function return statuses.
84
85    Build Mbed Crypto with -DRECORD_PSA_STATUS_COVERAGE_LOG, run the
86    test suites and display information about observed return statuses.
87    """
88    rebuilt = False
89    if not options.use_existing_log and os.path.exists(options.log_file):
90        os.remove(options.log_file)
91    if not os.path.exists(options.log_file):
92        if options.clean_before:
93            subprocess.check_call(['make', 'clean'],
94                                  cwd='tests',
95                                  stdout=sys.stderr)
96        with open(os.devnull, 'w') as devnull:
97            make_q_ret = subprocess.call(['make', '-q', 'lib', 'tests'],
98                                         stdout=devnull, stderr=devnull)
99        if make_q_ret != 0:
100            subprocess.check_call(['make', 'RECORD_PSA_STATUS_COVERAGE_LOG=1'],
101                                  stdout=sys.stderr)
102            rebuilt = True
103        subprocess.check_call(['make', 'test'],
104                              stdout=sys.stderr)
105    data = Statuses()
106    data.collect_log(options.log_file)
107    data.get_constant_names(options.psa_constant_names)
108    if rebuilt and options.clean_after:
109        subprocess.check_call(['make', 'clean'],
110                              cwd='tests',
111                              stdout=sys.stderr)
112    return data
113
114def main():
115    parser = argparse.ArgumentParser(description=globals()['__doc__'])
116    parser.add_argument('--clean-after',
117                        action='store_true',
118                        help='Run "make clean" after rebuilding')
119    parser.add_argument('--clean-before',
120                        action='store_true',
121                        help='Run "make clean" before regenerating the log file)')
122    parser.add_argument('--log-file', metavar='FILE',
123                        default=DEFAULT_STATUS_LOG_FILE,
124                        help='Log file location (default: {})'.format(
125                            DEFAULT_STATUS_LOG_FILE
126                        ))
127    parser.add_argument('--psa-constant-names', metavar='PROGRAM',
128                        default=DEFAULT_PSA_CONSTANT_NAMES,
129                        help='Path to psa_constant_names (default: {})'.format(
130                            DEFAULT_PSA_CONSTANT_NAMES
131                        ))
132    parser.add_argument('--use-existing-log', '-e',
133                        action='store_true',
134                        help='Don\'t regenerate the log file if it exists')
135    options = parser.parse_args()
136    data = collect_status_logs(options)
137    data.report()
138
139if __name__ == '__main__':
140    main()
141