1#!/usr/bin/env python3
2# See utils/checkpackagelib/readme.txt before editing this file.
3
4import argparse
5import inspect
6import magic
7import os
8import re
9import sys
10
11import checkpackagelib.base
12import checkpackagelib.lib_config
13import checkpackagelib.lib_defconfig
14import checkpackagelib.lib_hash
15import checkpackagelib.lib_ignore
16import checkpackagelib.lib_mk
17import checkpackagelib.lib_patch
18import checkpackagelib.lib_python
19import checkpackagelib.lib_shellscript
20import checkpackagelib.lib_sysv
21
22VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES = 3
23flags = None  # Command line arguments.
24
25# There are two Python packages called 'magic':
26#   https://pypi.org/project/file-magic/
27#   https://pypi.org/project/python-magic/
28# Both allow to return a MIME file type, but with a slightly different
29# interface. Detect which one of the two we have based on one of the
30# attributes.
31if hasattr(magic, 'FileMagic'):
32    # https://pypi.org/project/file-magic/
33    def get_filetype(fname):
34        return magic.detect_from_filename(fname).mime_type
35else:
36    # https://pypi.org/project/python-magic/
37    def get_filetype(fname):
38        return magic.from_file(fname, mime=True)
39
40
41def get_ignored_parsers_per_file(intree_only, ignore_filename):
42    ignored = dict()
43    entry_base_dir = ''
44
45    if not ignore_filename:
46        return ignored
47
48    filename = os.path.abspath(ignore_filename)
49    entry_base_dir = os.path.join(os.path.dirname(filename))
50
51    with open(filename, "r") as f:
52        for line in f.readlines():
53            filename, warnings_str = line.split(' ', 1)
54            warnings = warnings_str.split()
55            ignored[os.path.join(entry_base_dir, filename)] = warnings
56    return ignored
57
58
59def parse_args():
60    parser = argparse.ArgumentParser()
61
62    # Do not use argparse.FileType("r") here because only files with known
63    # format will be open based on the filename.
64    parser.add_argument("files", metavar="F", type=str, nargs="*",
65                        help="list of files")
66
67    parser.add_argument("--br2-external", "-b", dest='intree_only', action="store_false",
68                        help="do not apply the pathname filters used for intree files")
69    parser.add_argument("--ignore-list", dest='ignore_filename', action="store",
70                        help='override the default list of ignored warnings')
71
72    parser.add_argument("--manual-url", action="store",
73                        default="https://nightly.buildroot.org/",
74                        help="default: %(default)s")
75    parser.add_argument("--verbose", "-v", action="count", default=0)
76    parser.add_argument("--quiet", "-q", action="count", default=0)
77
78    # Now the debug options in the order they are processed.
79    parser.add_argument("--include-only", dest="include_list", action="append",
80                        help="run only the specified functions (debug)")
81    parser.add_argument("--exclude", dest="exclude_list", action="append",
82                        help="do not run the specified functions (debug)")
83    parser.add_argument("--dry-run", action="store_true", help="print the "
84                        "functions that would be called for each file (debug)")
85    parser.add_argument("--failed-only", action="store_true", help="print only"
86                        " the name of the functions that failed (debug)")
87
88    flags = parser.parse_args()
89
90    flags.ignore_list = get_ignored_parsers_per_file(flags.intree_only, flags.ignore_filename)
91
92    if flags.failed_only:
93        flags.dry_run = False
94        flags.verbose = -1
95
96    return flags
97
98
99def get_lib_from_filetype(fname):
100    if not os.path.isfile(fname):
101        return None
102    filetype = get_filetype(fname)
103    if filetype == "text/x-shellscript":
104        return checkpackagelib.lib_shellscript
105    if filetype in ["text/x-python", "text/x-script.python"]:
106        return checkpackagelib.lib_python
107    return None
108
109
110CONFIG_IN_FILENAME = re.compile(r"Config\.\S*$")
111DO_CHECK_INTREE = re.compile(r"|".join([
112    r".checkpackageignore",
113    r"Config.in",
114    r"arch/",
115    r"board/",
116    r"boot/",
117    r"configs/",
118    r"fs/",
119    r"linux/",
120    r"package/",
121    r"support/",
122    r"system/",
123    r"toolchain/",
124    r"utils/",
125    ]))
126DO_NOT_CHECK_INTREE = re.compile(r"|".join([
127    r"boot/barebox/barebox\.mk$",
128    r"fs/common\.mk$",
129    r"package/doc-asciidoc\.mk$",
130    r"package/pkg-\S*\.mk$",
131    r"support/dependencies/[^/]+\.mk$",
132    r"support/gnuconfig/config\.",
133    r"support/kconfig/",
134    r"support/misc/[^/]+\.mk$",
135    r"support/testing/tests/.*br2-external/",
136    r"toolchain/helpers\.mk$",
137    r"toolchain/toolchain-external/pkg-toolchain-external\.mk$",
138    ]))
139
140SYSV_INIT_SCRIPT_FILENAME = re.compile(r"/S\d\d[^/]+$")
141
142# For defconfigs: avoid matching kernel, uboot... defconfig files, so
143# limit to defconfig files in a configs/ directory, either in-tree or
144# in a br2-external tree.
145BR_DEFCONFIG_FILENAME = re.compile(r"^(.+/)?configs/[^/]+_defconfig$")
146
147
148def get_lib_from_filename(fname):
149    if flags.intree_only:
150        if DO_CHECK_INTREE.match(fname) is None:
151            return None
152        if DO_NOT_CHECK_INTREE.match(fname):
153            return None
154    else:
155        if os.path.basename(fname) == "external.mk" and \
156           os.path.exists(fname[:-2] + "desc"):
157            return None
158    if fname == ".checkpackageignore":
159        return checkpackagelib.lib_ignore
160    if CONFIG_IN_FILENAME.search(fname):
161        return checkpackagelib.lib_config
162    if BR_DEFCONFIG_FILENAME.search(fname):
163        return checkpackagelib.lib_defconfig
164    if fname.endswith(".hash"):
165        return checkpackagelib.lib_hash
166    if fname.endswith(".mk"):
167        return checkpackagelib.lib_mk
168    if fname.endswith(".patch"):
169        return checkpackagelib.lib_patch
170    if SYSV_INIT_SCRIPT_FILENAME.search(fname):
171        return checkpackagelib.lib_sysv
172    return get_lib_from_filetype(fname)
173
174
175def common_inspect_rules(m):
176    # do not call the base class
177    if m.__name__.startswith("_"):
178        return False
179    if flags.include_list and m.__name__ not in flags.include_list:
180        return False
181    if flags.exclude_list and m.__name__ in flags.exclude_list:
182        return False
183    return True
184
185
186def is_a_check_function(m):
187    if not inspect.isclass(m):
188        return False
189    if not issubclass(m, checkpackagelib.base._CheckFunction):
190        return False
191    return common_inspect_rules(m)
192
193
194def is_external_tool(m):
195    if not inspect.isclass(m):
196        return False
197    if not issubclass(m, checkpackagelib.base._Tool):
198        return False
199    return common_inspect_rules(m)
200
201
202def print_warnings(warnings, xfail):
203    # Avoid the need to use 'return []' at the end of every check function.
204    if warnings is None:
205        return 0, 0  # No warning generated.
206
207    if xfail:
208        return 0, 1  # Warning not generated, fail expected for this file.
209    for level, message in enumerate(warnings):
210        if flags.verbose >= level:
211            print(message.replace("\t", "< tab  >").rstrip())
212    return 1, 1  # One more warning to count.
213
214
215def check_file_using_lib(fname):
216    # Count number of warnings generated and lines processed.
217    nwarnings = 0
218    nlines = 0
219    xfail = flags.ignore_list.get(os.path.abspath(fname), [])
220    failed = set()
221
222    lib = get_lib_from_filename(fname)
223    if not lib:
224        if flags.verbose >= VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES:
225            print("{}: ignored".format(fname))
226        return nwarnings, nlines
227    internal_functions = inspect.getmembers(lib, is_a_check_function)
228    external_tools = inspect.getmembers(lib, is_external_tool)
229    all_checks = internal_functions + external_tools
230
231    if flags.dry_run:
232        functions_to_run = [c[0] for c in all_checks]
233        print("{}: would run: {}".format(fname, functions_to_run))
234        return nwarnings, nlines
235
236    objects = [[f"{lib.__name__[16:]}.{c[0]}", c[1](fname, flags.manual_url)] for c in internal_functions]
237
238    for name, cf in objects:
239        warn, fail = print_warnings(cf.before(), name in xfail)
240        if fail > 0:
241            failed.add(name)
242        nwarnings += warn
243
244    lastline = ""
245    with open(fname, "r", errors="surrogateescape") as f:
246        for lineno, text in enumerate(f):
247            nlines += 1
248            for name, cf in objects:
249                if cf.disable.search(lastline):
250                    continue
251                line_sts = cf.check_line(lineno + 1, text)
252                warn, fail = print_warnings(line_sts, name in xfail)
253                if fail > 0:
254                    failed.add(name)
255                nwarnings += warn
256            lastline = text
257
258    for name, cf in objects:
259        warn, fail = print_warnings(cf.after(), name in xfail)
260        if fail > 0:
261            failed.add(name)
262        nwarnings += warn
263
264    tools = [[c[0], c[1](fname)] for c in external_tools]
265
266    for name, tool in tools:
267        warn, fail = print_warnings(tool.run(), name in xfail)
268        if fail > 0:
269            failed.add(name)
270        nwarnings += warn
271
272    for should_fail in xfail:
273        if should_fail not in failed:
274            print("{}:0: {} was expected to fail, did you fix the file and forget to update {}?"
275                  .format(fname, should_fail, flags.ignore_filename))
276            nwarnings += 1
277
278    if flags.failed_only:
279        if len(failed) > 0:
280            f = " ".join(sorted(failed))
281            print("{} {}".format(fname, f))
282
283    return nwarnings, nlines
284
285
286def __main__():
287    global flags
288    flags = parse_args()
289
290    if flags.intree_only:
291        # change all paths received to be relative to the base dir
292        base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
293        files_to_check = [os.path.relpath(os.path.abspath(f), base_dir) for f in flags.files]
294        # move current dir so the script find the files
295        os.chdir(base_dir)
296    else:
297        files_to_check = flags.files
298
299    if len(files_to_check) == 0:
300        print("No files to check style")
301        sys.exit(1)
302
303    # Accumulate number of warnings generated and lines processed.
304    total_warnings = 0
305    total_lines = 0
306
307    for fname in files_to_check:
308        nwarnings, nlines = check_file_using_lib(fname)
309        total_warnings += nwarnings
310        total_lines += nlines
311
312    # The warning messages are printed to stdout and can be post-processed
313    # (e.g. counted by 'wc'), so for stats use stderr. Wait all warnings are
314    # printed, for the case there are many of them, before printing stats.
315    sys.stdout.flush()
316
317    if not flags.quiet:
318        print("{} lines processed".format(total_lines), file=sys.stderr)
319        print("{} warnings generated".format(total_warnings), file=sys.stderr)
320
321    if total_warnings > 0 and not flags.failed_only:
322        sys.exit(1)
323
324
325__main__()
326