1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2014 Google, Inc 3# 4 5"""Implementation the bulider threads 6 7This module provides the BuilderThread class, which handles calling the builder 8based on the jobs provided. 9""" 10 11import errno 12import glob 13import io 14import os 15import shutil 16import sys 17import threading 18 19from buildman import cfgutil 20from u_boot_pylib import command 21from u_boot_pylib import gitutil 22from u_boot_pylib import tools 23 24RETURN_CODE_RETRY = -1 25BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl'] 26 27# Common extensions for images 28COMMON_EXTS = ['.bin', '.rom', '.itb', '.img'] 29 30def mkdir(dirname, parents=False): 31 """Make a directory if it doesn't already exist. 32 33 Args: 34 dirname (str): Directory to create, or None to do nothing 35 parents (bool): True to also make parent directories 36 37 Raises: 38 OSError: File already exists 39 """ 40 if not dirname or os.path.exists(dirname): 41 return 42 try: 43 if parents: 44 os.makedirs(dirname) 45 else: 46 os.mkdir(dirname) 47 except OSError as err: 48 if err.errno == errno.EEXIST: 49 if os.path.realpath('.') == os.path.realpath(dirname): 50 raise ValueError( 51 f"Cannot create the current working directory '{dirname}'!") 52 else: 53 raise 54 55 56def _remove_old_outputs(out_dir): 57 """Remove any old output-target files 58 59 Args: 60 out_dir (str): Output directory for the build, or None for current dir 61 62 Since we use a build directory that was previously used by another 63 board, it may have produced an SPL image. If we don't remove it (i.e. 64 see do_config and self.mrproper below) then it will appear to be the 65 output of this build, even if it does not produce SPL images. 66 """ 67 for elf in BASE_ELF_FILENAMES: 68 fname = os.path.join(out_dir or '', elf) 69 if os.path.exists(fname): 70 os.remove(fname) 71 72 73def copy_files(out_dir, build_dir, dirname, patterns): 74 """Copy files from the build directory to the output. 75 76 Args: 77 out_dir (str): Path to output directory containing the files 78 build_dir (str): Place to copy the files 79 dirname (str): Source directory, '' for normal U-Boot, 'spl' for SPL 80 patterns (list of str): A list of filenames to copy, each relative 81 to the build directory 82 """ 83 for pattern in patterns: 84 file_list = glob.glob(os.path.join(out_dir, dirname, pattern)) 85 for fname in file_list: 86 target = os.path.basename(fname) 87 if dirname: 88 base, ext = os.path.splitext(target) 89 if ext: 90 target = f'{base}-{dirname}{ext}' 91 shutil.copy(fname, os.path.join(build_dir, target)) 92 93 94# pylint: disable=R0903 95class BuilderJob: 96 """Holds information about a job to be performed by a thread 97 98 Members: 99 brd: Board object to build 100 commits: List of Commit objects to build 101 keep_outputs: True to save build output files 102 step: 1 to process every commit, n to process every nth commit 103 work_in_output: Use the output directory as the work directory and 104 don't write to a separate output directory. 105 """ 106 def __init__(self): 107 self.brd = None 108 self.commits = [] 109 self.keep_outputs = False 110 self.step = 1 111 self.work_in_output = False 112 113 114class ResultThread(threading.Thread): 115 """This thread processes results from builder threads. 116 117 It simply passes the results on to the builder. There is only one 118 result thread, and this helps to serialise the build output. 119 """ 120 def __init__(self, builder): 121 """Set up a new result thread 122 123 Args: 124 builder: Builder which will be sent each result 125 """ 126 threading.Thread.__init__(self) 127 self.builder = builder 128 129 def run(self): 130 """Called to start up the result thread. 131 132 We collect the next result job and pass it on to the build. 133 """ 134 while True: 135 result = self.builder.out_queue.get() 136 self.builder.process_result(result) 137 self.builder.out_queue.task_done() 138 139 140class BuilderThread(threading.Thread): 141 """This thread builds U-Boot for a particular board. 142 143 An input queue provides each new job. We run 'make' to build U-Boot 144 and then pass the results on to the output queue. 145 146 Members: 147 builder: The builder which contains information we might need 148 thread_num: Our thread number (0-n-1), used to decide on a 149 temporary directory. If this is -1 then there are no threads 150 and we are the (only) main process 151 mrproper: Use 'make mrproper' before each reconfigure 152 per_board_out_dir: True to build in a separate persistent directory per 153 board rather than a thread-specific directory 154 test_exception: Used for testing; True to raise an exception instead of 155 reporting the build result 156 """ 157 def __init__(self, builder, thread_num, mrproper, per_board_out_dir, 158 test_exception=False): 159 """Set up a new builder thread""" 160 threading.Thread.__init__(self) 161 self.builder = builder 162 self.thread_num = thread_num 163 self.mrproper = mrproper 164 self.per_board_out_dir = per_board_out_dir 165 self.test_exception = test_exception 166 self.toolchain = None 167 168 def make(self, commit, brd, stage, cwd, *args, **kwargs): 169 """Run 'make' on a particular commit and board. 170 171 The source code will already be checked out, so the 'commit' 172 argument is only for information. 173 174 Args: 175 commit (Commit): Commit that is being built 176 brd (Board): Board that is being built 177 stage (str): Stage of the build. Valid stages are: 178 mrproper - can be called to clean source 179 config - called to configure for a board 180 build - the main make invocation - it does the build 181 cwd (str): Working directory to set, or None to leave it alone 182 *args (list of str): Arguments to pass to 'make' 183 **kwargs (dict): A list of keyword arguments to pass to 184 command.run_one() 185 186 Returns: 187 CommandResult object 188 """ 189 return self.builder.do_make(commit, brd, stage, cwd, *args, **kwargs) 190 191 def _build_args(self, brd, out_dir, out_rel_dir, work_dir, commit_upto): 192 """Set up arguments to the args list based on the settings 193 194 Args: 195 brd (Board): Board to create arguments for 196 out_dir (str): Path to output directory containing the files, or 197 or None to not use a separate output directory 198 out_rel_dir (str): Output directory relative to the current dir 199 work_dir (str): Directory to which the source will be checked out, 200 or None to use current directory 201 commit_upto (int): Commit number to build (0...n-1) 202 203 Returns: 204 tuple: 205 list of str: Arguments to pass to make 206 str: Current working directory, or None if no commit 207 str: Source directory (typically the work directory) 208 """ 209 args = [] 210 cwd = work_dir 211 src_dir = os.path.realpath(work_dir) if work_dir else os.getcwd() 212 if commit_upto is None: 213 # In this case we are building in the original source directory 214 # (i.e. the current directory where buildman is invoked. The 215 # output directory is set to this thread's selected work 216 # directory. 217 # 218 # Symlinks can confuse U-Boot's Makefile since we may use '..' 219 # in our path, so remove them. 220 if out_dir: 221 real_dir = os.path.realpath(out_dir) 222 args.append(f'O={real_dir}') 223 cwd = None 224 src_dir = os.getcwd() 225 elif out_rel_dir: 226 args.append(f'O={out_rel_dir}') 227 if self.builder.verbose_build: 228 args.append('V=1') 229 else: 230 args.append('-s') 231 if self.builder.num_jobs is not None: 232 args.extend(['-j', str(self.builder.num_jobs)]) 233 if self.builder.warnings_as_errors: 234 args.append('KCFLAGS=-Werror') 235 args.append('HOSTCFLAGS=-Werror') 236 if self.builder.allow_missing: 237 args.append('BINMAN_ALLOW_MISSING=1') 238 if self.builder.no_lto: 239 args.append('NO_LTO=1') 240 if self.builder.reproducible_builds: 241 args.append('SOURCE_DATE_EPOCH=0') 242 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 243 args.extend(self.toolchain.MakeArgs()) 244 return args, cwd, src_dir 245 246 def _reconfigure(self, commit, brd, cwd, args, env, config_args, config_out, 247 cmd_list, mrproper): 248 """Reconfigure the build 249 250 Args: 251 commit (Commit): Commit only being built 252 brd (Board): Board being built 253 cwd (str): Current working directory 254 args (list of str): Arguments to pass to make 255 env (dict): Environment strings 256 config_args (list of str): defconfig arg for this board 257 cmd_list (list of str): List to add the commands to, for logging 258 mrproper (bool): True to run mrproper first 259 260 Returns: 261 CommandResult object 262 """ 263 if mrproper: 264 result = self.make(commit, brd, 'mrproper', cwd, 'mrproper', *args, 265 env=env) 266 config_out.write(result.combined) 267 cmd_list.append([self.builder.gnu_make, 'mrproper', *args]) 268 result = self.make(commit, brd, 'config', cwd, *(args + config_args), 269 env=env) 270 cmd_list.append([self.builder.gnu_make] + args + config_args) 271 config_out.write(result.combined) 272 return result 273 274 def _build(self, commit, brd, cwd, args, env, cmd_list, config_only): 275 """Perform the build 276 277 Args: 278 commit (Commit): Commit only being built 279 brd (Board): Board being built 280 cwd (str): Current working directory 281 args (list of str): Arguments to pass to make 282 env (dict): Environment strings 283 cmd_list (list of str): List to add the commands to, for logging 284 config_only (bool): True if this is a config-only build (using the 285 'make cfg' target) 286 287 Returns: 288 CommandResult object 289 """ 290 if config_only: 291 args.append('cfg') 292 elif self.builder.build_target: 293 args.append(self.builder.build_target) 294 result = self.make(commit, brd, 'build', cwd, *args, env=env) 295 cmd_list.append([self.builder.gnu_make] + args) 296 if (result.return_code == 2 and 297 ('Some images are invalid' in result.stderr)): 298 # This is handled later by the check for output in stderr 299 result.return_code = 0 300 return result 301 302 def _read_done_file(self, commit_upto, brd, force_build, 303 force_build_failures): 304 """Check the 'done' file and see if this commit should be built 305 306 Args: 307 commit (Commit): Commit only being built 308 brd (Board): Board being built 309 force_build (bool): Force a build even if one was previously done 310 force_build_failures (bool): Force a bulid if the previous result 311 showed failure 312 313 Returns: 314 tuple: 315 bool: True if build should be built 316 CommandResult: if there was a previous run: 317 - already_done set to True 318 - return_code set to return code 319 - result.stderr set to 'bad' if stderr output was recorded 320 """ 321 result = command.CommandResult() 322 done_file = self.builder.get_done_file(commit_upto, brd.target) 323 result.already_done = os.path.exists(done_file) 324 will_build = (force_build or force_build_failures or 325 not result.already_done) 326 if result.already_done: 327 with open(done_file, 'r', encoding='utf-8') as outf: 328 try: 329 result.return_code = int(outf.readline()) 330 except ValueError: 331 # The file may be empty due to running out of disk space. 332 # Try a rebuild 333 result.return_code = RETURN_CODE_RETRY 334 335 # Check the signal that the build needs to be retried 336 if result.return_code == RETURN_CODE_RETRY: 337 will_build = True 338 elif will_build: 339 err_file = self.builder.get_err_file(commit_upto, brd.target) 340 if os.path.exists(err_file) and os.stat(err_file).st_size: 341 result.stderr = 'bad' 342 elif not force_build: 343 # The build passed, so no need to build it again 344 will_build = False 345 return will_build, result 346 347 def _decide_dirs(self, brd, work_dir, work_in_output): 348 """Decide the output directory to use 349 350 Args: 351 work_dir (str): Directory to which the source will be checked out 352 work_in_output (bool): Use the output directory as the work 353 directory and don't write to a separate output directory. 354 355 Returns: 356 tuple: 357 out_dir (str): Output directory for the build 358 out_rel_dir (str): Output directory relatie to the current dir 359 """ 360 if work_in_output or self.builder.in_tree: 361 out_rel_dir = None 362 out_dir = work_dir 363 else: 364 if self.per_board_out_dir: 365 out_rel_dir = os.path.join('..', brd.target) 366 else: 367 out_rel_dir = 'build' 368 out_dir = os.path.join(work_dir, out_rel_dir) 369 return out_dir, out_rel_dir 370 371 def _checkout(self, commit_upto, work_dir): 372 """Checkout the right commit 373 374 Args: 375 commit_upto (int): Commit number to build (0...n-1) 376 work_dir (str): Directory to which the source will be checked out 377 378 Returns: 379 Commit: Commit being built, or 'current' for current source 380 """ 381 if self.builder.commits: 382 commit = self.builder.commits[commit_upto] 383 if self.builder.checkout: 384 git_dir = os.path.join(work_dir, '.git') 385 gitutil.checkout(commit.hash, git_dir, work_dir, force=True) 386 else: 387 commit = 'current' 388 return commit 389 390 def _config_and_build(self, commit_upto, brd, work_dir, do_config, mrproper, 391 config_only, adjust_cfg, commit, out_dir, out_rel_dir, 392 result): 393 """Do the build, configuring first if necessary 394 395 Args: 396 commit_upto (int): Commit number to build (0...n-1) 397 brd (Board): Board to create arguments for 398 work_dir (str): Directory to which the source will be checked out 399 do_config (bool): True to run a make <board>_defconfig on the source 400 mrproper (bool): True to run mrproper first 401 config_only (bool): Only configure the source, do not build it 402 adjust_cfg (list of str): See the cfgutil module and run_commit() 403 commit (Commit): Commit only being built 404 out_dir (str): Output directory for the build, or None to use 405 current 406 out_rel_dir (str): Output directory relatie to the current dir 407 result (CommandResult): Previous result 408 409 Returns: 410 tuple: 411 result (CommandResult): Result of the build 412 do_config (bool): indicates whether 'make config' is needed on 413 the next incremental build 414 """ 415 # Set up the environment and command line 416 env = self.builder.make_environment(self.toolchain) 417 if out_dir and not os.path.exists(out_dir): 418 mkdir(out_dir) 419 420 args, cwd, src_dir = self._build_args(brd, out_dir, out_rel_dir, 421 work_dir, commit_upto) 422 config_args = [f'{brd.target}_defconfig'] 423 config_out = io.StringIO() 424 425 _remove_old_outputs(out_dir) 426 427 # If we need to reconfigure, do that now 428 cfg_file = os.path.join(out_dir or '', '.config') 429 cmd_list = [] 430 if do_config or adjust_cfg: 431 result = self._reconfigure( 432 commit, brd, cwd, args, env, config_args, config_out, cmd_list, 433 mrproper) 434 do_config = False # No need to configure next time 435 if adjust_cfg: 436 cfgutil.adjust_cfg_file(cfg_file, adjust_cfg) 437 438 # Now do the build, if everything looks OK 439 if result.return_code == 0: 440 if adjust_cfg: 441 oldc_args = list(args) + ['oldconfig'] 442 oldc_result = self.make(commit, brd, 'oldconfig', cwd, 443 *oldc_args, env=env) 444 if oldc_result.return_code: 445 return oldc_result 446 result = self._build(commit, brd, cwd, args, env, cmd_list, 447 config_only) 448 if adjust_cfg: 449 errs = cfgutil.check_cfg_file(cfg_file, adjust_cfg) 450 if errs: 451 result.stderr += errs 452 result.return_code = 1 453 result.stderr = result.stderr.replace(src_dir + '/', '') 454 if self.builder.verbose_build: 455 result.stdout = config_out.getvalue() + result.stdout 456 result.cmd_list = cmd_list 457 return result, do_config 458 459 def run_commit(self, commit_upto, brd, work_dir, do_config, mrproper, 460 config_only, force_build, force_build_failures, 461 work_in_output, adjust_cfg): 462 """Build a particular commit. 463 464 If the build is already done, and we are not forcing a build, we skip 465 the build and just return the previously-saved results. 466 467 Args: 468 commit_upto (int): Commit number to build (0...n-1) 469 brd (Board): Board to build 470 work_dir (str): Directory to which the source will be checked out 471 do_config (bool): True to run a make <board>_defconfig on the source 472 mrproper (bool): True to run mrproper first 473 config_only (bool): Only configure the source, do not build it 474 force_build (bool): Force a build even if one was previously done 475 force_build_failures (bool): Force a bulid if the previous result 476 showed failure 477 work_in_output (bool) : Use the output directory as the work 478 directory and don't write to a separate output directory. 479 adjust_cfg (list of str): List of changes to make to .config file 480 before building. Each is one of (where C is either CONFIG_xxx 481 or just xxx): 482 C to enable C 483 ~C to disable C 484 C=val to set the value of C (val must have quotes if C is 485 a string Kconfig 486 487 Returns: 488 tuple containing: 489 - CommandResult object containing the results of the build 490 - boolean indicating whether 'make config' is still needed 491 """ 492 # Create a default result - it will be overwritte by the call to 493 # self.make() below, in the event that we do a build. 494 out_dir, out_rel_dir = self._decide_dirs(brd, work_dir, work_in_output) 495 496 # Check if the job was already completed last time 497 will_build, result = self._read_done_file(commit_upto, brd, force_build, 498 force_build_failures) 499 500 if will_build: 501 # We are going to have to build it. First, get a toolchain 502 if not self.toolchain: 503 try: 504 self.toolchain = self.builder.toolchains.Select(brd.arch) 505 except ValueError as err: 506 result.return_code = 10 507 result.stdout = '' 508 result.stderr = f'Tool chain error for {brd.arch}: {str(err)}' 509 510 if self.toolchain: 511 commit = self._checkout(commit_upto, work_dir) 512 result, do_config = self._config_and_build( 513 commit_upto, brd, work_dir, do_config, mrproper, 514 config_only, adjust_cfg, commit, out_dir, out_rel_dir, 515 result) 516 result.already_done = False 517 518 result.toolchain = self.toolchain 519 result.brd = brd 520 result.commit_upto = commit_upto 521 result.out_dir = out_dir 522 return result, do_config 523 524 def _write_result(self, result, keep_outputs, work_in_output): 525 """Write a built result to the output directory. 526 527 Args: 528 result (CommandResult): result to write 529 keep_outputs (bool): True to store the output binaries, False 530 to delete them 531 work_in_output (bool): Use the output directory as the work 532 directory and don't write to a separate output directory. 533 """ 534 # If we think this might have been aborted with Ctrl-C, record the 535 # failure but not that we are 'done' with this board. A retry may fix 536 # it. 537 maybe_aborted = result.stderr and 'No child processes' in result.stderr 538 539 if result.return_code >= 0 and result.already_done: 540 return 541 542 # Write the output and stderr 543 output_dir = self.builder.get_output_dir(result.commit_upto) 544 mkdir(output_dir) 545 build_dir = self.builder.get_build_dir(result.commit_upto, 546 result.brd.target) 547 mkdir(build_dir) 548 549 outfile = os.path.join(build_dir, 'log') 550 with open(outfile, 'w', encoding='utf-8') as outf: 551 if result.stdout: 552 outf.write(result.stdout) 553 554 errfile = self.builder.get_err_file(result.commit_upto, 555 result.brd.target) 556 if result.stderr: 557 with open(errfile, 'w', encoding='utf-8') as outf: 558 outf.write(result.stderr) 559 elif os.path.exists(errfile): 560 os.remove(errfile) 561 562 # Fatal error 563 if result.return_code < 0: 564 return 565 566 done_file = self.builder.get_done_file(result.commit_upto, 567 result.brd.target) 568 if result.toolchain: 569 # Write the build result and toolchain information. 570 with open(done_file, 'w', encoding='utf-8') as outf: 571 if maybe_aborted: 572 # Special code to indicate we need to retry 573 outf.write(f'{RETURN_CODE_RETRY}') 574 else: 575 outf.write(f'{result.return_code}') 576 with open(os.path.join(build_dir, 'toolchain'), 'w', 577 encoding='utf-8') as outf: 578 print('gcc', result.toolchain.gcc, file=outf) 579 print('path', result.toolchain.path, file=outf) 580 print('cross', result.toolchain.cross, file=outf) 581 print('arch', result.toolchain.arch, file=outf) 582 outf.write(f'{result.return_code}') 583 584 # Write out the image and function size information and an objdump 585 env = self.builder.make_environment(self.toolchain) 586 with open(os.path.join(build_dir, 'out-env'), 'wb') as outf: 587 for var in sorted(env.keys()): 588 outf.write(b'%s="%s"' % (var, env[var])) 589 590 with open(os.path.join(build_dir, 'out-cmd'), 'w', 591 encoding='utf-8') as outf: 592 for cmd in result.cmd_list: 593 print(' '.join(cmd), file=outf) 594 595 lines = [] 596 for fname in BASE_ELF_FILENAMES: 597 cmd = [f'{self.toolchain.cross}nm', '--size-sort', fname] 598 nm_result = command.run_one(*cmd, capture=True, 599 capture_stderr=True, 600 cwd=result.out_dir, 601 raise_on_error=False, env=env) 602 if nm_result.stdout: 603 nm_fname = self.builder.get_func_sizes_file( 604 result.commit_upto, result.brd.target, fname) 605 with open(nm_fname, 'w', encoding='utf-8') as outf: 606 print(nm_result.stdout, end=' ', file=outf) 607 608 cmd = [f'{self.toolchain.cross}objdump', '-h', fname] 609 dump_result = command.run_one(*cmd, capture=True, 610 capture_stderr=True, 611 cwd=result.out_dir, 612 raise_on_error=False, env=env) 613 rodata_size = '' 614 if dump_result.stdout: 615 objdump = self.builder.get_objdump_file(result.commit_upto, 616 result.brd.target, fname) 617 with open(objdump, 'w', encoding='utf-8') as outf: 618 print(dump_result.stdout, end=' ', file=outf) 619 for line in dump_result.stdout.splitlines(): 620 fields = line.split() 621 if len(fields) > 5 and fields[1] == '.rodata': 622 rodata_size = fields[2] 623 624 cmd = [f'{self.toolchain.cross}size', fname] 625 size_result = command.run_one(*cmd, capture=True, 626 capture_stderr=True, 627 cwd=result.out_dir, 628 raise_on_error=False, env=env) 629 if size_result.stdout: 630 lines.append(size_result.stdout.splitlines()[1] + ' ' + 631 rodata_size) 632 633 # Extract the environment from U-Boot and dump it out 634 cmd = [f'{self.toolchain.cross}objcopy', '-O', 'binary', 635 '-j', '.rodata.default_environment', 636 'env/built-in.a', 'uboot.env'] 637 command.run_one(*cmd, capture=True, capture_stderr=True, 638 cwd=result.out_dir, raise_on_error=False, env=env) 639 if not work_in_output: 640 copy_files(result.out_dir, build_dir, '', ['uboot.env']) 641 642 # Write out the image sizes file. This is similar to the output 643 # of binutil's 'size' utility, but it omits the header line and 644 # adds an additional hex value at the end of each line for the 645 # rodata size 646 if lines: 647 sizes = self.builder.get_sizes_file(result.commit_upto, 648 result.brd.target) 649 with open(sizes, 'w', encoding='utf-8') as outf: 650 print('\n'.join(lines), file=outf) 651 else: 652 # Indicate that the build failure due to lack of toolchain 653 tools.write_file(done_file, '2\n', binary=False) 654 655 if not work_in_output: 656 # Write out the configuration files, with a special case for SPL 657 for dirname in ['', 'spl', 'tpl']: 658 copy_files( 659 result.out_dir, build_dir, dirname, 660 ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', 661 '.config', 'include/autoconf.mk', 662 'include/generated/autoconf.h']) 663 664 # Now write the actual build output 665 if keep_outputs: 666 to_copy = ['u-boot*', '*.map', 'MLO', 'SPL', 667 'include/autoconf.mk', 'spl/u-boot-spl*', 668 'tpl/u-boot-tpl*', 'vpl/u-boot-vpl*'] 669 to_copy += [f'*{ext}' for ext in COMMON_EXTS] 670 copy_files(result.out_dir, build_dir, '', to_copy) 671 672 def _send_result(self, result): 673 """Send a result to the builder for processing 674 675 Args: 676 result (CommandResult): results of the build 677 678 Raises: 679 ValueError: self.test_exception is true (for testing) 680 """ 681 if self.test_exception: 682 raise ValueError('test exception') 683 if self.thread_num != -1: 684 self.builder.out_queue.put(result) 685 else: 686 self.builder.process_result(result) 687 688 def run_job(self, job): 689 """Run a single job 690 691 A job consists of a building a list of commits for a particular board. 692 693 Args: 694 job (Job): Job to build 695 696 Raises: 697 ValueError: Thread was interrupted 698 """ 699 brd = job.brd 700 work_dir = self.builder.get_thread_dir(self.thread_num) 701 self.toolchain = None 702 if job.commits: 703 # Run 'make board_defconfig' on the first commit 704 do_config = True 705 commit_upto = 0 706 force_build = False 707 for commit_upto in range(0, len(job.commits), job.step): 708 result, request_config = self.run_commit(commit_upto, brd, 709 work_dir, do_config, self.mrproper, 710 self.builder.config_only, 711 force_build or self.builder.force_build, 712 self.builder.force_build_failures, 713 job.work_in_output, job.adjust_cfg) 714 failed = result.return_code or result.stderr 715 did_config = do_config 716 if failed and not do_config and not self.mrproper: 717 # If our incremental build failed, try building again 718 # with a reconfig. 719 if self.builder.force_config_on_failure: 720 result, request_config = self.run_commit(commit_upto, 721 brd, work_dir, True, 722 self.mrproper or self.builder.fallback_mrproper, 723 False, True, False, job.work_in_output, 724 job.adjust_cfg) 725 did_config = True 726 if not self.builder.force_reconfig: 727 do_config = request_config 728 729 # If we built that commit, then config is done. But if we got 730 # an warning, reconfig next time to force it to build the same 731 # files that created warnings this time. Otherwise an 732 # incremental build may not build the same file, and we will 733 # think that the warning has gone away. 734 # We could avoid this by using -Werror everywhere... 735 # For errors, the problem doesn't happen, since presumably 736 # the build stopped and didn't generate output, so will retry 737 # that file next time. So we could detect warnings and deal 738 # with them specially here. For now, we just reconfigure if 739 # anything goes work. 740 # Of course this is substantially slower if there are build 741 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 742 # have problems). 743 if (failed and not result.already_done and not did_config and 744 self.builder.force_config_on_failure): 745 # If this build failed, try the next one with a 746 # reconfigure. 747 # Sometimes if the board_config.h file changes it can mess 748 # with dependencies, and we get: 749 # make: *** No rule to make target `include/autoconf.mk', 750 # needed by `depend'. 751 do_config = True 752 force_build = True 753 else: 754 force_build = False 755 if self.builder.force_config_on_failure: 756 if failed: 757 do_config = True 758 result.commit_upto = commit_upto 759 if result.return_code < 0: 760 raise ValueError('Interrupt') 761 762 # We have the build results, so output the result 763 self._write_result(result, job.keep_outputs, job.work_in_output) 764 self._send_result(result) 765 else: 766 # Just build the currently checked-out build 767 result, request_config = self.run_commit(None, brd, work_dir, True, 768 self.mrproper, self.builder.config_only, True, 769 self.builder.force_build_failures, job.work_in_output, 770 job.adjust_cfg) 771 failed = result.return_code or result.stderr 772 if failed and not self.mrproper: 773 result, request_config = self.run_commit(None, brd, work_dir, 774 True, self.builder.fallback_mrproper, 775 self.builder.config_only, True, 776 self.builder.force_build_failures, 777 job.work_in_output, job.adjust_cfg) 778 779 result.commit_upto = 0 780 self._write_result(result, job.keep_outputs, job.work_in_output) 781 self._send_result(result) 782 783 def run(self): 784 """Our thread's run function 785 786 This thread picks a job from the queue, runs it, and then goes to the 787 next job. 788 """ 789 while True: 790 job = self.builder.queue.get() 791 try: 792 self.run_job(job) 793 except Exception as exc: 794 print('Thread exception (use -T0 to run without threads):', 795 exc) 796 self.builder.thread_exceptions.append(exc) 797 self.builder.queue.task_done() 798