1#!/usr/bin/env python3
2
3# Copyright (C) 2011 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
4# Copyright (C) 2013 by Yann E. MORIN <yann.morin.1998@free.fr>
5#
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; either version 2 of the License, or
9# (at your option) any later version.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14# General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
20# This script generates graphs of packages build time, from the timing
21# data generated by Buildroot in the $(O)/build-time.log file.
22#
23# Example usage:
24#
25#   cat $(O)/build-time.log | ./support/scripts/graph-build-time --type=histogram --output=foobar.pdf
26#
27# Three graph types are available :
28#
29#   * histogram, which creates an histogram of the build time for each
30#     package, decomposed by each step (extract, patch, configure,
31#     etc.). The order in which the packages are shown is
32#     configurable: by package name, by build order, or by duration
33#     order. See the --order option.
34#
35#   * pie-packages, which creates a pie chart of the build time of
36#     each package (without decomposition in steps). Packages that
37#     contributed to less than 1% of the overall build time are all
38#     grouped together in an "Other" entry.
39#
40#   * pie-steps, which creates a pie chart of the time spent globally
41#     on each step (extract, patch, configure, etc...)
42#
43# The default is to generate an histogram ordered by package name.
44#
45# Requirements:
46#
47#   * matplotlib (python-matplotlib on Debian/Ubuntu systems)
48#   * numpy (python-numpy on Debian/Ubuntu systems)
49#   * argparse (by default in Python 2.7, requires python-argparse if
50#     Python 2.6 is used)
51
52import sys
53
54try:
55    import matplotlib as mpl
56    import numpy
57except ImportError:
58    sys.stderr.write("You need python-matplotlib and python-numpy to generate build graphs\n")
59    exit(1)
60
61# Use the Agg backend (which produces a PNG output, see
62# http://matplotlib.org/faq/usage_faq.html#what-is-a-backend),
63# otherwise an incorrect backend is used on some host machines).
64# Note: matplotlib.use() must be called *before* matplotlib.pyplot.
65mpl.use('Agg')
66
67import matplotlib.pyplot as plt       # noqa: E402
68import matplotlib.font_manager as fm  # noqa: E402
69import csv                            # noqa: E402
70import argparse                       # noqa: E402
71
72steps = ['download', 'extract', 'patch', 'configure', 'build',
73         'install-target', 'install-staging', 'install-images',
74         'install-host']
75
76default_colors = ['#8d02ff', '#e60004', '#009836', '#2e1d86', '#ffed00',
77                  '#0068b5', '#f28e00', '#940084', '#97c000']
78
79alternate_colors = ['#ffbe0a', '#96bdff', '#3f7f7f', '#ff0000', '#00c000',
80                    '#0080ff', '#c000ff', '#00eeee', '#e0e000']
81
82
83class Package:
84    def __init__(self, name):
85        self.name = name
86        self.steps_duration = {}
87        self.steps_start = {}
88        self.steps_end = {}
89
90    def add_step(self, step, state, time):
91        if state == "start":
92            self.steps_start[step] = time
93        else:
94            self.steps_end[step] = time
95        if step in self.steps_start and step in self.steps_end:
96            self.steps_duration[step] = self.steps_end[step] - self.steps_start[step]
97
98    def get_duration(self, step=None):
99        if step is None:
100            duration = 0
101            for step in list(self.steps_duration.keys()):
102                duration += self.steps_duration[step]
103            return duration
104        if step in self.steps_duration:
105            return self.steps_duration[step]
106        return 0
107
108
109# Generate an histogram of the time spent in each step of each
110# package.
111def pkg_histogram(data, output, order="build"):
112    n_pkgs = len(data)
113    ind = numpy.arange(n_pkgs)
114
115    if order == "duration":
116        data = sorted(data, key=lambda p: p.get_duration(), reverse=True)
117    elif order == "name":
118        data = sorted(data, key=lambda p: p.name, reverse=False)
119
120    # Prepare the vals array, containing one entry for each step
121    vals = []
122    for step in steps:
123        val = []
124        for p in data:
125            val.append(p.get_duration(step))
126        vals.append(val)
127
128    bottom = [0] * n_pkgs
129    legenditems = []
130
131    plt.figure()
132
133    # Draw the bars, step by step
134    for i in range(0, len(vals)):
135        b = plt.bar(ind+0.1, vals[i], width=0.8, color=colors[i], bottom=bottom, linewidth=0.25)
136        legenditems.append(b[0])
137        bottom = [bottom[j] + vals[i][j] for j in range(0, len(vals[i]))]
138
139    # Draw the package names
140    plt.xticks(ind + .6, [p.name for p in data], rotation=-60, rotation_mode="anchor", fontsize=8, ha='left')
141
142    # Adjust size of graph depending on the number of packages
143    # Ensure a minimal size twice as the default
144    # Magic Numbers do Magic Layout!
145    ratio = max(((n_pkgs + 10) / 48, 2))
146    borders = 0.1 / ratio
147    sz = plt.gcf().get_figwidth()
148    plt.gcf().set_figwidth(sz * ratio)
149
150    # Adjust space at borders, add more space for the
151    # package names at the bottom
152    plt.gcf().subplots_adjust(bottom=0.2, left=borders, right=1-borders)
153
154    # Remove ticks in the graph for each package
155    axes = plt.gcf().gca()
156    for line in axes.get_xticklines():
157        line.set_markersize(0)
158
159    axes.set_ylabel('Time (seconds)')
160
161    # Reduce size of legend text
162    leg_prop = fm.FontProperties(size=6)
163
164    # Draw legend
165    plt.legend(legenditems, steps, prop=leg_prop)
166
167    if order == "name":
168        plt.title('Build time of packages\n')
169    elif order == "build":
170        plt.title('Build time of packages, by build order\n')
171    elif order == "duration":
172        plt.title('Build time of packages, by duration order\n')
173
174    # Save graph
175    plt.savefig(output)
176
177
178# Generate a pie chart with the time spent building each package.
179def pkg_pie_time_per_package(data, output):
180    # Compute total build duration
181    total = 0
182    for p in data:
183        total += p.get_duration()
184
185    # Build the list of labels and values, and filter the packages
186    # that account for less than 1% of the build time.
187    labels = []
188    values = []
189    other_value = 0
190    for p in sorted(data, key=lambda p: p.get_duration()):
191        if p.get_duration() < (total * 0.01):
192            other_value += p.get_duration()
193        else:
194            labels.append(p.name)
195            values.append(p.get_duration())
196
197    labels.append('Other')
198    values.append(other_value)
199
200    plt.figure()
201
202    # Draw pie graph
203    patches, texts, autotexts = plt.pie(values, labels=labels,
204                                        autopct='%1.1f%%', shadow=True,
205                                        colors=colors)
206
207    # Reduce text size
208    proptease = fm.FontProperties()
209    proptease.set_size('xx-small')
210    plt.setp(autotexts, fontproperties=proptease)
211    plt.setp(texts, fontproperties=proptease)
212
213    plt.title('Build time per package')
214    plt.savefig(output)
215
216
217# Generate a pie chart with a portion for the overall time spent in
218# each step for all packages.
219def pkg_pie_time_per_step(data, output):
220    steps_values = []
221    for step in steps:
222        val = 0
223        for p in data:
224            val += p.get_duration(step)
225        steps_values.append(val)
226
227    plt.figure()
228
229    # Draw pie graph
230    patches, texts, autotexts = plt.pie(steps_values, labels=steps,
231                                        autopct='%1.1f%%', shadow=True,
232                                        colors=colors)
233
234    # Reduce text size
235    proptease = fm.FontProperties()
236    proptease.set_size('xx-small')
237    plt.setp(autotexts, fontproperties=proptease)
238    plt.setp(texts, fontproperties=proptease)
239
240    plt.title('Build time per step')
241    plt.savefig(output)
242
243
244def pkg_timeline(data, output):
245    start = 0
246    end = 0
247
248    # Find the first timestamp and the last timestamp
249    for p in data:
250        for k, v in p.steps_start.items():
251            if start == 0 or v < start:
252                start = v
253            if end < v:
254                end = v
255
256    # Readjust all timestamps so that 0 is the start of the build
257    # instead of being Epoch
258    for p in data:
259        for k, v in p.steps_start.items():
260            p.steps_start[k] = v - start
261        for k, v in p.steps_end.items():
262            p.steps_end[k] = v - start
263
264    plt.figure()
265
266    i = 0
267    labels_names = []
268    labels_coords = []
269    # put last packages that started to configure last; this does not
270    # give the proper dependency chain, but still provides a good-enough
271    # cascade graph.
272    for p in sorted(data, reverse=True, key=lambda x: x.steps_start['configure']):
273        durations = []
274        facecolors = []
275        for step in steps:
276            if step not in p.steps_start or step not in p.steps_end:
277                continue
278            durations.append((p.steps_start[step],
279                              p.steps_end[step] - p.steps_start[step]))
280            facecolors.append(colors[steps.index(step)])
281        plt.broken_barh(durations, (i, 6), facecolors=facecolors)
282        labels_coords.append(i + 3)
283        labels_names.append(p.name)
284        i += 10
285
286    axes = plt.gcf().gca()
287
288    axes.set_ylim(0, i + 10)
289    axes.set_xlim(0, end - start)
290    axes.set_xlabel('seconds since start')
291    axes.set_yticks(labels_coords)
292    axes.set_yticklabels(labels_names)
293    axes.set_axisbelow(True)
294    axes.grid(True, linewidth=0.2, zorder=-1)
295
296    plt.gcf().subplots_adjust(left=0.2)
297
298    plt.tick_params(axis='y', which='both', labelsize=6)
299    plt.title('Timeline')
300    plt.savefig(output, dpi=300)
301
302
303# Parses the csv file passed on standard input and returns a list of
304# Package objects, filed with the duration of each step and the total
305# duration of the package.
306def read_data(input_file):
307    if input_file is None:
308        input_file = sys.stdin
309    else:
310        input_file = open(input_file)
311    reader = csv.reader(input_file, delimiter=':')
312    pkgs = []
313
314    # Auxilliary function to find a package by name in the list.
315    def getpkg(name):
316        for p in pkgs:
317            if p.name == name:
318                return p
319        return None
320
321    for row in reader:
322        time = float(row[0].strip())
323        state = row[1].strip()
324        step = row[2].strip()
325        pkg = row[3].strip()
326
327        p = getpkg(pkg)
328        if p is None:
329            p = Package(pkg)
330            pkgs.append(p)
331
332        p.add_step(step, state, time)
333
334    return pkgs
335
336
337parser = argparse.ArgumentParser(description='Draw build time graphs')
338parser.add_argument("--type", '-t', metavar="GRAPH_TYPE",
339                    help="Type of graph (histogram, pie-packages, pie-steps, timeline)")
340parser.add_argument("--order", '-O', metavar="GRAPH_ORDER",
341                    help="Ordering of packages: build or duration (for histogram only)")
342parser.add_argument("--alternate-colors", '-c', action="store_true",
343                    help="Use alternate colour-scheme")
344parser.add_argument("--input", '-i', metavar="INPUT",
345                    help="Input file (usually $(O)/build/build-time.log)")
346parser.add_argument("--output", '-o', metavar="OUTPUT", required=True,
347                    help="Output file (.pdf or .png extension)")
348args = parser.parse_args()
349
350d = read_data(args.input)
351
352if args.alternate_colors:
353    colors = alternate_colors
354else:
355    colors = default_colors
356
357if args.type == "histogram" or args.type is None:
358    if args.order == "build" or args.order == "duration" or args.order == "name":
359        pkg_histogram(d, args.output, args.order)
360    elif args.order is None:
361        pkg_histogram(d, args.output, "name")
362    else:
363        sys.stderr.write("Unknown ordering: %s\n" % args.order)
364        exit(1)
365elif args.type == "pie-packages":
366    pkg_pie_time_per_package(d, args.output)
367elif args.type == "pie-steps":
368    pkg_pie_time_per_step(d, args.output)
369elif args.type == "timeline":
370    pkg_timeline(d, args.output)
371else:
372    sys.stderr.write("Unknown type: %s\n" % args.type)
373    exit(1)
374