1#!/usr/bin/env python
2"""A wrapper script around clang-format, suitable for linting multiple files
3and to use for continuous integration.
4
5This is an alternative API for the clang-format command line.
6It runs over multiple files and directories in parallel.
7A diff output is produced and a sensible exit code is returned.
8
9"""
10
11from __future__ import print_function, unicode_literals
12
13import argparse
14import codecs
15import difflib
16import fnmatch
17import io
18import errno
19import multiprocessing
20import os
21import signal
22import subprocess
23import sys
24import traceback
25import platform
26
27from functools import partial
28
29try:
30    from subprocess import DEVNULL  # py3k
31except ImportError:
32    DEVNULL = open(os.devnull, "wb")
33
34
35DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx"
36DEFAULT_CLANG_FORMAT_IGNORE = ".clang-format-ignore"
37
38
39class ExitStatus:
40    SUCCESS = 0
41    DIFF = 1
42    TROUBLE = 2
43
44
45def excludes_from_file(ignore_file):
46    excludes = []
47    try:
48        with io.open(ignore_file, "r", encoding="utf-8") as f:
49            for line in f:
50                if line.startswith("#"):
51                    # ignore comments
52                    continue
53                pattern = line.rstrip()
54                if not pattern:
55                    # allow empty lines
56                    continue
57                excludes.append(pattern)
58    except EnvironmentError as e:
59        if e.errno != errno.ENOENT:
60            raise
61    return excludes
62
63
64def list_files(files, recursive=False, extensions=None, exclude=None):
65    if extensions is None:
66        extensions = []
67    if exclude is None:
68        exclude = []
69
70    out = []
71    for file in files:
72        if recursive and os.path.isdir(file):
73            for dirpath, dnames, fnames in os.walk(file):
74                fpaths = [
75                    os.path.relpath(os.path.join(dirpath, fname), os.getcwd())
76                    for fname in fnames
77                ]
78                for pattern in exclude:
79                    # os.walk() supports trimming down the dnames list
80                    # by modifying it in-place,
81                    # to avoid unnecessary directory listings.
82                    dnames[:] = [
83                        x
84                        for x in dnames
85                        if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)
86                    ]
87                    fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)]
88
89                for f in fpaths:
90                    ext = os.path.splitext(f)[1][1:]
91                    if ext in extensions:
92                        out.append(f)
93        else:
94            out.append(file)
95    return out
96
97
98def make_diff(file, original, reformatted):
99    return list(
100        difflib.unified_diff(
101            original,
102            reformatted,
103            fromfile="{}\t(original)".format(file),
104            tofile="{}\t(reformatted)".format(file),
105            n=3,
106        )
107    )
108
109
110class DiffError(Exception):
111    def __init__(self, message, errs=None):
112        super(DiffError, self).__init__(message)
113        self.errs = errs or []
114
115
116class UnexpectedError(Exception):
117    def __init__(self, message, exc=None):
118        super(UnexpectedError, self).__init__(message)
119        self.formatted_traceback = traceback.format_exc()
120        self.exc = exc
121
122
123def run_clang_format_diff_wrapper(args, file):
124    try:
125        ret = run_clang_format_diff(args, file)
126        return ret
127    except DiffError:
128        raise
129    except Exception as e:
130        raise UnexpectedError("{}: {}: {}".format(file, e.__class__.__name__, e), e)
131
132
133def run_clang_format_diff(args, file):
134    # try:
135    #     with io.open(file, "r", encoding="utf-8") as f:
136    #         original = f.readlines()
137    # except IOError as exc:
138    #     raise DiffError(str(exc))
139
140    if args.in_place:
141        invocation = [args.clang_format_executable, "-i", file]
142    else:
143        invocation = [args.clang_format_executable, file]
144
145    if args.style:
146        invocation.extend(["--style", args.style])
147
148    if args.dry_run:
149        print(" ".join(invocation))
150        return [], []
151
152    # Use of utf-8 to decode the process output.
153    #
154    # Hopefully, this is the correct thing to do.
155    #
156    # It's done due to the following assumptions (which may be incorrect):
157    # - clang-format will returns the bytes read from the files as-is,
158    #   without conversion, and it is already assumed that the files use utf-8.
159    # - if the diagnostics were internationalized, they would use utf-8:
160    #   > Adding Translations to Clang
161    #   >
162    #   > Not possible yet!
163    #   > Diagnostic strings should be written in UTF-8,
164    #   > the client can translate to the relevant code page if needed.
165    #   > Each translation completely replaces the format string
166    #   > for the diagnostic.
167    #   > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation
168    #
169    # It's not pretty, due to Python 2 & 3 compatibility.
170    encoding_py3 = {}
171    if sys.version_info[0] >= 3:
172        encoding_py3["encoding"] = "utf-8"
173
174    try:
175        proc = subprocess.Popen(
176            invocation,
177            stdout=subprocess.PIPE,
178            stderr=subprocess.PIPE,
179            universal_newlines=True,
180            **encoding_py3
181        )
182    except OSError as exc:
183        raise DiffError(
184            "Command '{}' failed to start: {}".format(
185                subprocess.list2cmdline(invocation), exc
186            )
187        )
188    proc_stdout = proc.stdout
189    proc_stderr = proc.stderr
190    if sys.version_info[0] < 3:
191        # make the pipes compatible with Python 3,
192        # reading lines should output unicode
193        encoding = "utf-8"
194        proc_stdout = codecs.getreader(encoding)(proc_stdout)
195        proc_stderr = codecs.getreader(encoding)(proc_stderr)
196    # hopefully the stderr pipe won't get full and block the process
197    outs = list(proc_stdout.readlines())
198    errs = list(proc_stderr.readlines())
199    proc.wait()
200    if proc.returncode:
201        raise DiffError(
202            "Command '{}' returned non-zero exit status {}".format(
203                subprocess.list2cmdline(invocation), proc.returncode
204            ),
205            errs,
206        )
207    if args.in_place:
208        return [], errs
209    return make_diff(file, original, outs), errs
210
211
212def bold_red(s):
213    return "\x1b[1m\x1b[31m" + s + "\x1b[0m"
214
215
216def colorize(diff_lines):
217    def bold(s):
218        return "\x1b[1m" + s + "\x1b[0m"
219
220    def cyan(s):
221        return "\x1b[36m" + s + "\x1b[0m"
222
223    def green(s):
224        return "\x1b[32m" + s + "\x1b[0m"
225
226    def red(s):
227        return "\x1b[31m" + s + "\x1b[0m"
228
229    for line in diff_lines:
230        if line[:4] in ["--- ", "+++ "]:
231            yield bold(line)
232        elif line.startswith("@@ "):
233            yield cyan(line)
234        elif line.startswith("+"):
235            yield green(line)
236        elif line.startswith("-"):
237            yield red(line)
238        else:
239            yield line
240
241
242def print_diff(diff_lines, use_color):
243    if use_color:
244        diff_lines = colorize(diff_lines)
245    if sys.version_info[0] < 3:
246        sys.stdout.writelines((l.encode("utf-8") for l in diff_lines))
247    else:
248        sys.stdout.writelines(diff_lines)
249
250
251def print_trouble(prog, message, use_colors):
252    error_text = "error:"
253    if use_colors:
254        error_text = bold_red(error_text)
255    print("{}: {} {}".format(prog, error_text, message), file=sys.stderr)
256
257
258def main():
259    parser = argparse.ArgumentParser(description=__doc__)
260    parser.add_argument(
261        "--clang-format-executable",
262        metavar="EXECUTABLE",
263        help="path to the clang-format executable",
264        default="clang-format",
265    )
266    parser.add_argument(
267        "--extensions",
268        help="comma separated list of file extensions (default: {})".format(
269            DEFAULT_EXTENSIONS
270        ),
271        default=DEFAULT_EXTENSIONS,
272    )
273    parser.add_argument(
274        "-r",
275        "--recursive",
276        action="store_true",
277        help="run recursively over directories",
278    )
279    parser.add_argument(
280        "-d", "--dry-run", action="store_true", help="just print the list of files"
281    )
282    parser.add_argument(
283        "-i",
284        "--in-place",
285        action="store_true",
286        help="format file instead of printing differences",
287    )
288    parser.add_argument("files", metavar="file", nargs="+")
289    parser.add_argument(
290        "-q",
291        "--quiet",
292        action="store_true",
293        help="disable output, useful for the exit code",
294    )
295    parser.add_argument(
296        "-j",
297        metavar="N",
298        type=int,
299        default=0,
300        help="run N clang-format jobs in parallel" " (default number of cpus + 1)",
301    )
302    parser.add_argument(
303        "--color",
304        default="auto",
305        choices=["auto", "always", "never"],
306        help="show colored diff (default: auto)",
307    )
308    parser.add_argument(
309        "-e",
310        "--exclude",
311        metavar="PATTERN",
312        action="append",
313        default=[],
314        help="exclude paths matching the given glob-like pattern(s)"
315        " from recursive search",
316    )
317    parser.add_argument(
318        "--style",
319        default="file",
320        help="formatting style to apply (LLVM, Google, Chromium, Mozilla, WebKit)",
321    )
322
323    args = parser.parse_args()
324
325    # use default signal handling, like diff return SIGINT value on ^C
326    # https://bugs.python.org/issue14229#msg156446
327    signal.signal(signal.SIGINT, signal.SIG_DFL)
328    try:
329        signal.SIGPIPE
330    except AttributeError:
331        # compatibility, SIGPIPE does not exist on Windows
332        pass
333    else:
334        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
335
336    colored_stdout = False
337    colored_stderr = False
338    if args.color == "always":
339        colored_stdout = True
340        colored_stderr = True
341    elif args.color == "auto":
342        colored_stdout = sys.stdout.isatty()
343        colored_stderr = sys.stderr.isatty()
344
345    version_invocation = [args.clang_format_executable, str("--version")]
346    try:
347        subprocess.check_call(version_invocation, stdout=DEVNULL)
348    except subprocess.CalledProcessError as e:
349        print_trouble(parser.prog, str(e), use_colors=colored_stderr)
350        return ExitStatus.TROUBLE
351    except OSError as e:
352        print_trouble(
353            parser.prog,
354            "Command '{}' failed to start: {}".format(
355                subprocess.list2cmdline(version_invocation), e
356            ),
357            use_colors=colored_stderr,
358        )
359        return ExitStatus.TROUBLE
360
361    retcode = ExitStatus.SUCCESS
362
363    if os.path.exists(DEFAULT_CLANG_FORMAT_IGNORE):
364        excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE)
365    else:
366        excludes = []
367    excludes.extend(args.exclude)
368
369    files = list_files(
370        args.files,
371        recursive=args.recursive,
372        exclude=excludes,
373        extensions=args.extensions.split(","),
374    )
375
376    if not files:
377        return
378
379    njobs = args.j
380    if njobs == 0:
381        njobs = multiprocessing.cpu_count() + 1
382    njobs = min(len(files), njobs)
383
384    if njobs == 1:
385        # execute directly instead of in a pool,
386        # less overhead, simpler stacktraces
387        it = (run_clang_format_diff_wrapper(args, file) for file in files)
388        pool = None
389    else:
390        pool = multiprocessing.Pool(njobs)
391        it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files)
392        pool.close()
393    while True:
394        try:
395            outs, errs = next(it)
396        except StopIteration:
397            break
398        except DiffError as e:
399            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
400            retcode = ExitStatus.TROUBLE
401            sys.stderr.writelines(e.errs)
402        except UnexpectedError as e:
403            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
404            sys.stderr.write(e.formatted_traceback)
405            retcode = ExitStatus.TROUBLE
406            # stop at the first unexpected error,
407            # something could be very wrong,
408            # don't process all files unnecessarily
409            if pool:
410                pool.terminate()
411            break
412        else:
413            sys.stderr.writelines(errs)
414            if outs == []:
415                continue
416            if not args.quiet:
417                print_diff(outs, use_color=colored_stdout)
418            if retcode == ExitStatus.SUCCESS:
419                retcode = ExitStatus.DIFF
420    if pool:
421        pool.join()
422    return retcode
423
424
425if __name__ == "__main__":
426    sys.exit(main())
427