1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0
3#
4# Copyright (C) Google LLC, 2018
5#
6# Author: Tom Roeder <tmroeder@google.com>
7# Ported and modified for U-Boot by Joao Marcos Costa <jmcosta944@gmail.com>
8# Briefly documented at doc/build/gen_compile_commands.rst
9#
10# Ported and modified for OP-TEE by Joakim Bech <joakim.bech@linaro.org>
11"""A tool for generating compile_commands.json in OP-TEE."""
12
13import argparse
14import json
15import logging
16import os
17import re
18import subprocess
19import sys
20
21_DEFAULT_OUTPUT = 'compile_commands.json'
22_DEFAULT_LOG_LEVEL = 'WARNING'
23
24_FILENAME_PATTERN = r'^\..*\.cmd$'
25_LINE_PATTERN = r'^old-cmd[^ ]* := (?:/usr/bin/ccache )?(.*) -c (\S+)'
26
27_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
28# The tools/ directory adopts a different build system, and produces .cmd
29# files in a different format. Do not support it.
30_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
31
32
33def parse_arguments():
34    """Sets up and parses command-line arguments.
35
36    Returns:
37        log_level: A logging level to filter log output.
38        directory: The work directory where the objects were built.
39        ar: Command used for parsing .a archives.
40        output: Where to write the compile-commands JSON file.
41        paths: The list of files/directories to handle to find .cmd files.
42    """
43    usage = 'Creates a compile_commands.json database from OP-TEE .cmd files'
44    parser = argparse.ArgumentParser(description=usage)
45
46    directory_help = ('specify the output directory used for the OP-TEE build '
47                      '(defaults to the working directory)')
48    parser.add_argument('-d', '--directory', type=str, default='.',
49                        help=directory_help)
50
51    output_help = ('path to the output command database (defaults to ' +
52                   _DEFAULT_OUTPUT + ')')
53    parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
54                        help=output_help)
55
56    log_level_help = ('the level of log messages to produce (defaults to ' +
57                      _DEFAULT_LOG_LEVEL + ')')
58    parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
59                        default=_DEFAULT_LOG_LEVEL, help=log_level_help)
60
61    ar_help = 'command used for parsing .a archives'
62    parser.add_argument('-a', '--ar', type=str, default='llvm-ar',
63                        help=ar_help)
64
65    paths_help = ('directories to search or files to parse '
66                  '(files should be *.o, *.a, or modules.order). '
67                  'If nothing is specified, the current directory is searched')
68    parser.add_argument('paths', type=str, nargs='*', help=paths_help)
69
70    args = parser.parse_args()
71
72    return (args.log_level,
73            os.path.abspath(args.directory),
74            args.output,
75            args.ar,
76            args.paths if len(args.paths) > 0 else [args.directory])
77
78
79def cmdfiles_in_dir(directory):
80    """Generate the iterator of .cmd files found under the directory.
81
82    Walk under the given directory, and yield every .cmd file found.
83
84    Args:
85        directory: The directory to search for .cmd files.
86
87    Yields:
88        The path to a .cmd file.
89    """
90
91    filename_matcher = re.compile(_FILENAME_PATTERN)
92    exclude_dirs = [os.path.join(directory, d) for d in _EXCLUDE_DIRS]
93
94    for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
95        # Prune unwanted directories.
96        if dirpath in exclude_dirs:
97            dirnames[:] = []
98            continue
99
100        for filename in filenames:
101            if filename_matcher.match(filename):
102                yield os.path.join(dirpath, filename)
103
104
105def to_cmdfile(path):
106    """Return the path of .cmd file used for the given build artifact
107
108    Args:
109        Path: file path
110
111    Returns:
112        The path to .cmd file
113    """
114    dir, base = os.path.split(path)
115    return os.path.join(dir, '.' + base + '.cmd')
116
117
118def cmdfiles_for_a(archive, ar):
119    """Generate the iterator of .cmd files associated with the archive.
120
121    Parse the given archive, and yield every .cmd file used to build it.
122
123    Args:
124        archive: The archive to parse
125
126    Yields:
127        The path to every .cmd file found
128    """
129    for obj in subprocess.check_output([ar, '-t', archive]).decode().split():
130        yield to_cmdfile(obj)
131
132
133def cmdfiles_for_modorder(modorder):
134    """Generate the iterator of .cmd files associated with the modules.order.
135
136    Parse the given modules.order, and yield every .cmd file used to build the
137    contained modules.
138
139    Args:
140        modorder: The modules.order file to parse
141
142    Yields:
143        The path to every .cmd file found
144    """
145    with open(modorder) as f:
146        for line in f:
147            obj = line.rstrip()
148            base, ext = os.path.splitext(obj)
149            if ext != '.o':
150                sys.exit('{}: module path must end with .o'.format(obj))
151            mod = base + '.mod'
152            # Read from *.mod, to get a list of objects that compose the
153            # module.
154            with open(mod) as m:
155                for mod_line in m:
156                    yield to_cmdfile(mod_line.rstrip())
157
158
159def process_line(root_directory, command_prefix, file_path):
160    """Extracts information from a .cmd line and creates an entry from it.
161
162    Args:
163        root_directory: The directory that was searched for .cmd files. Usually
164            used directly in the "directory" entry in compile_commands.json.
165        command_prefix: The extracted command line, up to the last element.
166        file_path: The .c file from the end of the extracted command.
167            Usually relative to root_directory, but sometimes absolute.
168
169    Returns:
170        An entry to append to compile_commands.
171
172    Raises:
173        ValueError: Could not find the extracted file based on file_path and
174            root_directory or file_directory.
175    """
176    # The .cmd files are intended to be included directly by Make, so they
177    # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
178    # kernel version). The compile_commands.json file is not interpreted
179    # by Make, so this code replaces the escaped version with '#'.
180    prefix = command_prefix.replace(r'\#', '#').replace('$(pound)', '#')
181
182    # Use os.path.abspath() to normalize the path resolving '.' and '..' .
183    abs_path = os.path.abspath(os.path.join(root_directory, file_path))
184    if not os.path.exists(abs_path):
185        raise ValueError('File %s not found' % abs_path)
186    return {
187        'directory': root_directory,
188        'file': abs_path,
189        'command': prefix + file_path,
190    }
191
192
193def main():
194    """Walks through the directory and finds and parses .cmd files."""
195    log_level, directory, output, ar, paths = parse_arguments()
196
197    level = getattr(logging, log_level)
198    logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
199
200    line_matcher = re.compile(_LINE_PATTERN)
201
202    compile_commands = []
203
204    for path in paths:
205        # If 'path' is a directory, handle all .cmd files under it.
206        # Otherwise, handle .cmd files associated with the file.
207        # built-in objects are linked via vmlinux.a
208        # Modules are listed in modules.order.
209        if os.path.isdir(path):
210            cmdfiles = cmdfiles_in_dir(path)
211        elif path.endswith('.a'):
212            cmdfiles = cmdfiles_for_a(path, ar)
213        elif path.endswith('modules.order'):
214            cmdfiles = cmdfiles_for_modorder(path)
215        else:
216            sys.exit('{}: unknown file type'.format(path))
217
218        for cmdfile in cmdfiles:
219            with open(cmdfile, 'rt') as f:
220                try:
221                    result = line_matcher.match(f.readline())
222                    if result:
223                        entry = process_line(directory, result.group(1),
224                                             result.group(2))
225                        compile_commands.append(entry)
226                except ValueError as err:
227                    logging.info('Could not add line from %s: %s',
228                                 cmdfile, err)
229                except AttributeError as err:
230                    continue
231
232    with open(output, 'wt') as f:
233        json.dump(compile_commands, f, indent=2, sort_keys=True)
234
235
236if __name__ == '__main__':
237    main()
238