1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2012 The Chromium OS Authors.
3# Author: Simon Glass <sjg@chromium.org>
4# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5
6"""Maintains a list of boards and allows them to be selected"""
7
8from collections import OrderedDict
9import errno
10import fnmatch
11import glob
12import multiprocessing
13import os
14import re
15import sys
16import tempfile
17import time
18
19from buildman import board
20from buildman import kconfiglib
21
22
23### constant variables ###
24OUTPUT_FILE = 'boards.cfg'
25CONFIG_DIR = 'configs'
26SLEEP_TIME = 0.03
27COMMENT_BLOCK = f'''#
28# List of boards
29#   Automatically generated by {__file__}: don't edit
30#
31# Status, Arch, CPU, SoC, Vendor, Board, Target, Config, Maintainers
32
33'''
34
35
36def try_remove(fname):
37    """Remove a file ignoring 'No such file or directory' error.
38
39    Args:
40        fname (str): Filename to remove
41
42    Raises:
43        OSError: output file exists but could not be removed
44    """
45    try:
46        os.remove(fname)
47    except OSError as exception:
48        # Ignore 'No such file or directory' error
49        if exception.errno != errno.ENOENT:
50            raise
51
52
53def output_is_new(output):
54    """Check if the output file is up to date.
55
56    Looks at defconfig and Kconfig files to make sure none is newer than the
57    output file. Also ensures that the boards.cfg does not mention any removed
58    boards.
59
60    Args:
61        output (str): Filename to check
62
63    Returns:
64        True if the given output file exists and is newer than any of
65        *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
66
67    Raises:
68        OSError: output file exists but could not be opened
69    """
70    # pylint: disable=too-many-branches
71    try:
72        ctime = os.path.getctime(output)
73    except OSError as exception:
74        if exception.errno == errno.ENOENT:
75            # return False on 'No such file or directory' error
76            return False
77        raise
78
79    for (dirpath, _, filenames) in os.walk(CONFIG_DIR):
80        for filename in fnmatch.filter(filenames, '*_defconfig'):
81            if fnmatch.fnmatch(filename, '.*'):
82                continue
83            filepath = os.path.join(dirpath, filename)
84            if ctime < os.path.getctime(filepath):
85                return False
86
87    for (dirpath, _, filenames) in os.walk('.'):
88        for filename in filenames:
89            if (fnmatch.fnmatch(filename, '*~') or
90                not fnmatch.fnmatch(filename, 'Kconfig*') and
91                not filename == 'MAINTAINERS'):
92                continue
93            filepath = os.path.join(dirpath, filename)
94            if ctime < os.path.getctime(filepath):
95                return False
96
97    # Detect a board that has been removed since the current board database
98    # was generated
99    with open(output, encoding="utf-8") as inf:
100        for line in inf:
101            if 'Options,' in line:
102                return False
103            if line[0] == '#' or line == '\n':
104                continue
105            defconfig = line.split()[6] + '_defconfig'
106            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
107                return False
108
109    return True
110
111
112class Expr:
113    """A single regular expression for matching boards to build"""
114
115    def __init__(self, expr):
116        """Set up a new Expr object.
117
118        Args:
119            expr (str): String cotaining regular expression to store
120        """
121        self._expr = expr
122        self._re = re.compile(expr)
123
124    def matches(self, props):
125        """Check if any of the properties match the regular expression.
126
127        Args:
128           props (list of str): List of properties to check
129        Returns:
130           True if any of the properties match the regular expression
131        """
132        for prop in props:
133            if self._re.match(prop):
134                return True
135        return False
136
137    def __str__(self):
138        return self._expr
139
140class Term:
141    """A list of expressions each of which must match with properties.
142
143    This provides a list of 'AND' expressions, meaning that each must
144    match the board properties for that board to be built.
145    """
146    def __init__(self):
147        self._expr_list = []
148        self._board_count = 0
149
150    def add_expr(self, expr):
151        """Add an Expr object to the list to check.
152
153        Args:
154            expr (Expr): New Expr object to add to the list of those that must
155                  match for a board to be built.
156        """
157        self._expr_list.append(Expr(expr))
158
159    def __str__(self):
160        """Return some sort of useful string describing the term"""
161        return '&'.join([str(expr) for expr in self._expr_list])
162
163    def matches(self, props):
164        """Check if any of the properties match this term
165
166        Each of the expressions in the term is checked. All must match.
167
168        Args:
169           props (list of str): List of properties to check
170        Returns:
171           True if all of the expressions in the Term match, else False
172        """
173        for expr in self._expr_list:
174            if not expr.matches(props):
175                return False
176        return True
177
178
179class KconfigScanner:
180
181    """Kconfig scanner."""
182
183    ### constant variable only used in this class ###
184    _SYMBOL_TABLE = {
185        'arch' : 'SYS_ARCH',
186        'cpu' : 'SYS_CPU',
187        'soc' : 'SYS_SOC',
188        'vendor' : 'SYS_VENDOR',
189        'board' : 'SYS_BOARD',
190        'config' : 'SYS_CONFIG_NAME',
191        # 'target' is added later
192    }
193
194    def __init__(self):
195        """Scan all the Kconfig files and create a Kconfig object."""
196        # Define environment variables referenced from Kconfig
197        os.environ['srctree'] = os.getcwd()
198        os.environ['UBOOTVERSION'] = 'dummy'
199        os.environ['KCONFIG_OBJDIR'] = ''
200        self._tmpfile = None
201        self._conf = kconfiglib.Kconfig(warn=False)
202
203    def __del__(self):
204        """Delete a leftover temporary file before exit.
205
206        The scan() method of this class creates a temporay file and deletes
207        it on success.  If scan() method throws an exception on the way,
208        the temporary file might be left over.  In that case, it should be
209        deleted in this destructor.
210        """
211        if self._tmpfile:
212            try_remove(self._tmpfile)
213
214    def scan(self, defconfig):
215        """Load a defconfig file to obtain board parameters.
216
217        Args:
218            defconfig (str): path to the defconfig file to be processed
219
220        Returns:
221            A dictionary of board parameters.  It has a form of:
222            {
223                'arch': <arch_name>,
224                'cpu': <cpu_name>,
225                'soc': <soc_name>,
226                'vendor': <vendor_name>,
227                'board': <board_name>,
228                'target': <target_name>,
229                'config': <config_header_name>,
230            }
231        """
232        # strip special prefixes and save it in a temporary file
233        outfd, self._tmpfile = tempfile.mkstemp()
234        with os.fdopen(outfd, 'w') as outf:
235            with open(defconfig, encoding='utf-8') as inf:
236                for line in inf:
237                    colon = line.find(':CONFIG_')
238                    if colon == -1:
239                        outf.write(line)
240                    else:
241                        outf.write(line[colon + 1:])
242
243        self._conf.load_config(self._tmpfile)
244        try_remove(self._tmpfile)
245        self._tmpfile = None
246
247        params = {}
248
249        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
250        # Set '-' if the value is empty.
251        for key, symbol in list(self._SYMBOL_TABLE.items()):
252            value = self._conf.syms.get(symbol).str_value
253            if value:
254                params[key] = value
255            else:
256                params[key] = '-'
257
258        defconfig = os.path.basename(defconfig)
259        params['target'], match, rear = defconfig.partition('_defconfig')
260        assert match and not rear, f'{defconfig} : invalid defconfig'
261
262        # fix-up for aarch64
263        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
264            params['arch'] = 'aarch64'
265
266        # fix-up for riscv
267        if params['arch'] == 'riscv':
268            try:
269                value = self._conf.syms.get('ARCH_RV32I').str_value
270            except:
271                value = ''
272            if value == 'y':
273                params['arch'] = 'riscv32'
274            else:
275                params['arch'] = 'riscv64'
276
277        return params
278
279
280class MaintainersDatabase:
281
282    """The database of board status and maintainers.
283
284    Properties:
285        database: dict:
286            key: Board-target name (e.g. 'snow')
287            value: tuple:
288                str: Board status (e.g. 'Active')
289                str: List of maintainers, separated by :
290        warnings (list of str): List of warnings due to missing status, etc.
291    """
292
293    def __init__(self):
294        """Create an empty database."""
295        self.database = {}
296        self.warnings = []
297
298    def get_status(self, target):
299        """Return the status of the given board.
300
301        The board status is generally either 'Active' or 'Orphan'.
302        Display a warning message and return '-' if status information
303        is not found.
304
305        Args:
306            target (str): Build-target name
307
308        Returns:
309            str: 'Active', 'Orphan' or '-'.
310        """
311        if not target in self.database:
312            self.warnings.append(f"WARNING: no status info for '{target}'")
313            return '-'
314
315        tmp = self.database[target][0]
316        if tmp.startswith('Maintained'):
317            return 'Active'
318        if tmp.startswith('Supported'):
319            return 'Active'
320        if tmp.startswith('Orphan'):
321            return 'Orphan'
322        self.warnings.append(f"WARNING: {tmp}: unknown status for '{target}'")
323        return '-'
324
325    def get_maintainers(self, target):
326        """Return the maintainers of the given board.
327
328        Args:
329            target (str): Build-target name
330
331        Returns:
332            str: Maintainers of the board.  If the board has two or more
333            maintainers, they are separated with colons.
334        """
335        if not target in self.database:
336            self.warnings.append(f"WARNING: no maintainers for '{target}'")
337            return ''
338
339        return ':'.join(self.database[target][1])
340
341    def parse_file(self, fname):
342        """Parse a MAINTAINERS file.
343
344        Parse a MAINTAINERS file and accumulate board status and maintainers
345        information in the self.database dict.
346
347        Args:
348            fname (str): MAINTAINERS file to be parsed
349        """
350        targets = []
351        maintainers = []
352        status = '-'
353        with open(fname, encoding="utf-8") as inf:
354            for line in inf:
355                # Check also commented maintainers
356                if line[:3] == '#M:':
357                    line = line[1:]
358                tag, rest = line[:2], line[2:].strip()
359                if tag == 'M:':
360                    maintainers.append(rest)
361                elif tag == 'F:':
362                    # expand wildcard and filter by 'configs/*_defconfig'
363                    for item in glob.glob(rest):
364                        front, match, rear = item.partition('configs/')
365                        if not front and match:
366                            front, match, rear = rear.rpartition('_defconfig')
367                            if match and not rear:
368                                targets.append(front)
369                elif tag == 'S:':
370                    status = rest
371                elif tag == 'N:':
372                    # Just scan the configs directory since that's all we care
373                    # about
374                    for dirpath, _, fnames in os.walk('configs'):
375                        for fname in fnames:
376                            path = os.path.join(dirpath, fname)
377                            front, match, rear = path.partition('configs/')
378                            if not front and match:
379                                front, match, rear = rear.rpartition('_defconfig')
380                                if match and not rear:
381                                    targets.append(front)
382                elif line == '\n':
383                    for target in targets:
384                        self.database[target] = (status, maintainers)
385                    targets = []
386                    maintainers = []
387                    status = '-'
388        if targets:
389            for target in targets:
390                self.database[target] = (status, maintainers)
391
392
393class Boards:
394    """Manage a list of boards."""
395    def __init__(self):
396        self._boards = []
397
398    def add_board(self, brd):
399        """Add a new board to the list.
400
401        The board's target member must not already exist in the board list.
402
403        Args:
404            brd (Board): board to add
405        """
406        self._boards.append(brd)
407
408    def read_boards(self, fname):
409        """Read a list of boards from a board file.
410
411        Create a Board object for each and add it to our _boards list.
412
413        Args:
414            fname (str): Filename of boards.cfg file
415        """
416        with open(fname, 'r', encoding='utf-8') as inf:
417            for line in inf:
418                if line[0] == '#':
419                    continue
420                fields = line.split()
421                if not fields:
422                    continue
423                for upto, field in enumerate(fields):
424                    if field == '-':
425                        fields[upto] = ''
426                while len(fields) < 8:
427                    fields.append('')
428                if len(fields) > 8:
429                    fields = fields[:8]
430
431                brd = board.Board(*fields)
432                self.add_board(brd)
433
434
435    def get_list(self):
436        """Return a list of available boards.
437
438        Returns:
439            List of Board objects
440        """
441        return self._boards
442
443    def get_dict(self):
444        """Build a dictionary containing all the boards.
445
446        Returns:
447            Dictionary:
448                key is board.target
449                value is board
450        """
451        board_dict = OrderedDict()
452        for brd in self._boards:
453            board_dict[brd.target] = brd
454        return board_dict
455
456    def get_selected_dict(self):
457        """Return a dictionary containing the selected boards
458
459        Returns:
460            List of Board objects that are marked selected
461        """
462        board_dict = OrderedDict()
463        for brd in self._boards:
464            if brd.build_it:
465                board_dict[brd.target] = brd
466        return board_dict
467
468    def get_selected(self):
469        """Return a list of selected boards
470
471        Returns:
472            List of Board objects that are marked selected
473        """
474        return [brd for brd in self._boards if brd.build_it]
475
476    def get_selected_names(self):
477        """Return a list of selected boards
478
479        Returns:
480            List of board names that are marked selected
481        """
482        return [brd.target for brd in self._boards if brd.build_it]
483
484    @classmethod
485    def _build_terms(cls, args):
486        """Convert command line arguments to a list of terms.
487
488        This deals with parsing of the arguments. It handles the '&'
489        operator, which joins several expressions into a single Term.
490
491        For example:
492            ['arm & freescale sandbox', 'tegra']
493
494        will produce 3 Terms containing expressions as follows:
495            arm, freescale
496            sandbox
497            tegra
498
499        The first Term has two expressions, both of which must match for
500        a board to be selected.
501
502        Args:
503            args (list of str): List of command line arguments
504
505        Returns:
506            list of Term: A list of Term objects
507        """
508        syms = []
509        for arg in args:
510            for word in arg.split():
511                sym_build = []
512                for term in word.split('&'):
513                    if term:
514                        sym_build.append(term)
515                    sym_build.append('&')
516                syms += sym_build[:-1]
517        terms = []
518        term = None
519        oper = None
520        for sym in syms:
521            if sym == '&':
522                oper = sym
523            elif oper:
524                term.add_expr(sym)
525                oper = None
526            else:
527                if term:
528                    terms.append(term)
529                term = Term()
530                term.add_expr(sym)
531        if term:
532            terms.append(term)
533        return terms
534
535    def select_boards(self, args, exclude=None, brds=None):
536        """Mark boards selected based on args
537
538        Normally either boards (an explicit list of boards) or args (a list of
539        terms to match against) is used. It is possible to specify both, in
540        which case they are additive.
541
542        If brds and args are both empty, all boards are selected.
543
544        Args:
545            args (list of str): List of strings specifying boards to include,
546                either named, or by their target, architecture, cpu, vendor or
547                soc. If empty, all boards are selected.
548            exclude (list of str): List of boards to exclude, regardless of
549                'args', or None for none
550            brds (list of Board): List of boards to build, or None/[] for all
551
552        Returns:
553            Tuple
554                Dictionary which holds the list of boards which were selected
555                    due to each argument, arranged by argument.
556                List of errors found
557        """
558        def _check_board(brd):
559            """Check whether to include or exclude a board
560
561            Checks the various terms and decide whether to build it or not (the
562            'build_it' variable).
563
564            If it is built, add the board to the result[term] list so we know
565            which term caused it to be built. Add it to result['all'] also.
566
567            Keep a list of boards we found in 'found', so we can report boards
568            which appear in self._boards but not in brds.
569
570            Args:
571                brd (Board): Board to check
572            """
573            matching_term = None
574            build_it = False
575            if terms:
576                for term in terms:
577                    if term.matches(brd.props):
578                        matching_term = str(term)
579                        build_it = True
580                        break
581            elif brds:
582                if brd.target in brds:
583                    build_it = True
584                    found.append(brd.target)
585            else:
586                build_it = True
587
588            # Check that it is not specifically excluded
589            for expr in exclude_list:
590                if expr.matches(brd.props):
591                    build_it = False
592                    break
593
594            if build_it:
595                brd.build_it = True
596                if matching_term:
597                    result[matching_term].append(brd.target)
598                result['all'].append(brd.target)
599
600        result = OrderedDict()
601        warnings = []
602        terms = self._build_terms(args)
603
604        result['all'] = []
605        for term in terms:
606            result[str(term)] = []
607
608        exclude_list = []
609        if exclude:
610            for expr in exclude:
611                exclude_list.append(Expr(expr))
612
613        found = []
614        for brd in self._boards:
615            _check_board(brd)
616
617        if brds:
618            remaining = set(brds) - set(found)
619            if remaining:
620                warnings.append(f"Boards not found: {', '.join(remaining)}\n")
621
622        return result, warnings
623
624    @classmethod
625    def scan_defconfigs_for_multiprocess(cls, queue, defconfigs):
626        """Scan defconfig files and queue their board parameters
627
628        This function is intended to be passed to multiprocessing.Process()
629        constructor.
630
631        Args:
632            queue (multiprocessing.Queue): The resulting board parameters are
633                written into this.
634            defconfigs (sequence of str): A sequence of defconfig files to be
635                scanned.
636        """
637        kconf_scanner = KconfigScanner()
638        for defconfig in defconfigs:
639            queue.put(kconf_scanner.scan(defconfig))
640
641    @classmethod
642    def read_queues(cls, queues, params_list):
643        """Read the queues and append the data to the paramers list"""
644        for que in queues:
645            while not que.empty():
646                params_list.append(que.get())
647
648    def scan_defconfigs(self, jobs=1):
649        """Collect board parameters for all defconfig files.
650
651        This function invokes multiple processes for faster processing.
652
653        Args:
654            jobs (int): The number of jobs to run simultaneously
655        """
656        all_defconfigs = []
657        for (dirpath, _, filenames) in os.walk(CONFIG_DIR):
658            for filename in fnmatch.filter(filenames, '*_defconfig'):
659                if fnmatch.fnmatch(filename, '.*'):
660                    continue
661                all_defconfigs.append(os.path.join(dirpath, filename))
662
663        total_boards = len(all_defconfigs)
664        processes = []
665        queues = []
666        for i in range(jobs):
667            defconfigs = all_defconfigs[total_boards * i // jobs :
668                                        total_boards * (i + 1) // jobs]
669            que = multiprocessing.Queue(maxsize=-1)
670            proc = multiprocessing.Process(
671                target=self.scan_defconfigs_for_multiprocess,
672                args=(que, defconfigs))
673            proc.start()
674            processes.append(proc)
675            queues.append(que)
676
677        # The resulting data should be accumulated to this list
678        params_list = []
679
680        # Data in the queues should be retrieved preriodically.
681        # Otherwise, the queues would become full and subprocesses would get stuck.
682        while any(p.is_alive() for p in processes):
683            self.read_queues(queues, params_list)
684            # sleep for a while until the queues are filled
685            time.sleep(SLEEP_TIME)
686
687        # Joining subprocesses just in case
688        # (All subprocesses should already have been finished)
689        for proc in processes:
690            proc.join()
691
692        # retrieve leftover data
693        self.read_queues(queues, params_list)
694
695        return params_list
696
697    @classmethod
698    def insert_maintainers_info(cls, params_list):
699        """Add Status and Maintainers information to the board parameters list.
700
701        Args:
702            params_list (list of dict): A list of the board parameters
703
704        Returns:
705            list of str: List of warnings collected due to missing status, etc.
706        """
707        database = MaintainersDatabase()
708        for (dirpath, _, filenames) in os.walk('.'):
709            if 'MAINTAINERS' in filenames:
710                database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
711
712        for i, params in enumerate(params_list):
713            target = params['target']
714            params['status'] = database.get_status(target)
715            params['maintainers'] = database.get_maintainers(target)
716            params_list[i] = params
717        return database.warnings
718
719    @classmethod
720    def format_and_output(cls, params_list, output):
721        """Write board parameters into a file.
722
723        Columnate the board parameters, sort lines alphabetically,
724        and then write them to a file.
725
726        Args:
727            params_list (list of dict): The list of board parameters
728            output (str): The path to the output file
729        """
730        fields = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
731                  'config', 'maintainers')
732
733        # First, decide the width of each column
734        max_length = {f: 0 for f in fields}
735        for params in params_list:
736            for field in fields:
737                max_length[field] = max(max_length[field], len(params[field]))
738
739        output_lines = []
740        for params in params_list:
741            line = ''
742            for field in fields:
743                # insert two spaces between fields like column -t would
744                line += '  ' + params[field].ljust(max_length[field])
745            output_lines.append(line.strip())
746
747        # ignore case when sorting
748        output_lines.sort(key=str.lower)
749
750        with open(output, 'w', encoding="utf-8") as outf:
751            outf.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
752
753    def ensure_board_list(self, output, jobs=1, force=False, quiet=False):
754        """Generate a board database file if needed.
755
756        Args:
757            output (str): The name of the output file
758            jobs (int): The number of jobs to run simultaneously
759            force (bool): Force to generate the output even if it is new
760            quiet (bool): True to avoid printing a message if nothing needs doing
761
762        Returns:
763            bool: True if all is well, False if there were warnings
764        """
765        if not force and output_is_new(output):
766            if not quiet:
767                print(f'{output} is up to date. Nothing to do.')
768            return True
769        params_list = self.scan_defconfigs(jobs)
770        warnings = self.insert_maintainers_info(params_list)
771        for warn in warnings:
772            print(warn, file=sys.stderr)
773        self.format_and_output(params_list, output)
774        return not warnings
775