1#!/usr/bin/env python3
2
3"""Sanity checks for test data.
4
5This program contains a class for traversing test cases that can be used
6independently of the checks.
7"""
8
9# Copyright The Mbed TLS Contributors
10# SPDX-License-Identifier: Apache-2.0
11#
12# Licensed under the Apache License, Version 2.0 (the "License"); you may
13# not use this file except in compliance with the License.
14# You may obtain a copy of the License at
15#
16# http://www.apache.org/licenses/LICENSE-2.0
17#
18# Unless required by applicable law or agreed to in writing, software
19# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
20# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21# See the License for the specific language governing permissions and
22# limitations under the License.
23
24import argparse
25import glob
26import os
27import re
28import sys
29
30class Results:
31    """Store file and line information about errors or warnings in test suites."""
32
33    def __init__(self, options):
34        self.errors = 0
35        self.warnings = 0
36        self.ignore_warnings = options.quiet
37
38    def error(self, file_name, line_number, fmt, *args):
39        sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
40                         format(file_name, line_number, *args))
41        self.errors += 1
42
43    def warning(self, file_name, line_number, fmt, *args):
44        if not self.ignore_warnings:
45            sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
46                             .format(file_name, line_number, *args))
47            self.warnings += 1
48
49class TestDescriptionExplorer:
50    """An iterator over test cases with descriptions.
51
52The test cases that have descriptions are:
53* Individual unit tests (entries in a .data file) in test suites.
54* Individual test cases in ssl-opt.sh.
55
56This is an abstract class. To use it, derive a class that implements
57the process_test_case method, and call walk_all().
58"""
59
60    def process_test_case(self, per_file_state,
61                          file_name, line_number, description):
62        """Process a test case.
63
64per_file_state: an object created by new_per_file_state() at the beginning
65                of each file.
66file_name: a relative path to the file containing the test case.
67line_number: the line number in the given file.
68description: the test case description as a byte string.
69"""
70        raise NotImplementedError
71
72    def new_per_file_state(self):
73        """Return a new per-file state object.
74
75The default per-file state object is None. Child classes that require per-file
76state may override this method.
77"""
78        #pylint: disable=no-self-use
79        return None
80
81    def walk_test_suite(self, data_file_name):
82        """Iterate over the test cases in the given unit test data file."""
83        in_paragraph = False
84        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
85        with open(data_file_name, 'rb') as data_file:
86            for line_number, line in enumerate(data_file, 1):
87                line = line.rstrip(b'\r\n')
88                if not line:
89                    in_paragraph = False
90                    continue
91                if line.startswith(b'#'):
92                    continue
93                if not in_paragraph:
94                    # This is a test case description line.
95                    self.process_test_case(descriptions,
96                                           data_file_name, line_number, line)
97                in_paragraph = True
98
99    def walk_ssl_opt_sh(self, file_name):
100        """Iterate over the test cases in ssl-opt.sh or a file with a similar format."""
101        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
102        with open(file_name, 'rb') as file_contents:
103            for line_number, line in enumerate(file_contents, 1):
104                # Assume that all run_test calls have the same simple form
105                # with the test description entirely on the same line as the
106                # function name.
107                m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line)
108                if not m:
109                    continue
110                description = m.group(1)
111                self.process_test_case(descriptions,
112                                       file_name, line_number, description)
113
114    @staticmethod
115    def collect_test_directories():
116        """Get the relative path for the TLS and Crypto test directories."""
117        if os.path.isdir('tests'):
118            tests_dir = 'tests'
119        elif os.path.isdir('suites'):
120            tests_dir = '.'
121        elif os.path.isdir('../suites'):
122            tests_dir = '..'
123        directories = [tests_dir]
124        return directories
125
126    def walk_all(self):
127        """Iterate over all named test cases."""
128        test_directories = self.collect_test_directories()
129        for directory in test_directories:
130            for data_file_name in glob.glob(os.path.join(directory, 'suites',
131                                                         '*.data')):
132                self.walk_test_suite(data_file_name)
133            ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
134            if os.path.exists(ssl_opt_sh):
135                self.walk_ssl_opt_sh(ssl_opt_sh)
136
137class DescriptionChecker(TestDescriptionExplorer):
138    """Check all test case descriptions.
139
140* Check that each description is valid (length, allowed character set, etc.).
141* Check that there is no duplicated description inside of one test suite.
142"""
143
144    def __init__(self, results):
145        self.results = results
146
147    def new_per_file_state(self):
148        """Dictionary mapping descriptions to their line number."""
149        return {}
150
151    def process_test_case(self, per_file_state,
152                          file_name, line_number, description):
153        """Check test case descriptions for errors."""
154        results = self.results
155        seen = per_file_state
156        if description in seen:
157            results.error(file_name, line_number,
158                          'Duplicate description (also line {})',
159                          seen[description])
160            return
161        if re.search(br'[\t;]', description):
162            results.error(file_name, line_number,
163                          'Forbidden character \'{}\' in description',
164                          re.search(br'[\t;]', description).group(0).decode('ascii'))
165        if re.search(br'[^ -~]', description):
166            results.error(file_name, line_number,
167                          'Non-ASCII character in description')
168        if len(description) > 66:
169            results.warning(file_name, line_number,
170                            'Test description too long ({} > 66)',
171                            len(description))
172        seen[description] = line_number
173
174def main():
175    parser = argparse.ArgumentParser(description=__doc__)
176    parser.add_argument('--quiet', '-q',
177                        action='store_true',
178                        help='Hide warnings')
179    parser.add_argument('--verbose', '-v',
180                        action='store_false', dest='quiet',
181                        help='Show warnings (default: on; undoes --quiet)')
182    options = parser.parse_args()
183    results = Results(options)
184    checker = DescriptionChecker(results)
185    checker.walk_all()
186    if (results.warnings or results.errors) and not options.quiet:
187        sys.stderr.write('{}: {} errors, {} warnings\n'
188                         .format(sys.argv[0], results.errors, results.warnings))
189    sys.exit(1 if results.errors else 0)
190
191if __name__ == '__main__':
192    main()
193