1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2014 Google, Inc 3# 4 5import errno 6import glob 7import os 8import shutil 9import sys 10import threading 11 12from buildman import cfgutil 13from patman import gitutil 14from u_boot_pylib import command 15 16RETURN_CODE_RETRY = -1 17BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl'] 18 19def Mkdir(dirname, parents = False): 20 """Make a directory if it doesn't already exist. 21 22 Args: 23 dirname: Directory to create 24 """ 25 try: 26 if parents: 27 os.makedirs(dirname) 28 else: 29 os.mkdir(dirname) 30 except OSError as err: 31 if err.errno == errno.EEXIST: 32 if os.path.realpath('.') == os.path.realpath(dirname): 33 print("Cannot create the current working directory '%s'!" % dirname) 34 sys.exit(1) 35 pass 36 else: 37 raise 38 39class BuilderJob: 40 """Holds information about a job to be performed by a thread 41 42 Members: 43 brd: Board object to build 44 commits: List of Commit objects to build 45 keep_outputs: True to save build output files 46 step: 1 to process every commit, n to process every nth commit 47 work_in_output: Use the output directory as the work directory and 48 don't write to a separate output directory. 49 """ 50 def __init__(self): 51 self.brd = None 52 self.commits = [] 53 self.keep_outputs = False 54 self.step = 1 55 self.work_in_output = False 56 57 58class ResultThread(threading.Thread): 59 """This thread processes results from builder threads. 60 61 It simply passes the results on to the builder. There is only one 62 result thread, and this helps to serialise the build output. 63 """ 64 def __init__(self, builder): 65 """Set up a new result thread 66 67 Args: 68 builder: Builder which will be sent each result 69 """ 70 threading.Thread.__init__(self) 71 self.builder = builder 72 73 def run(self): 74 """Called to start up the result thread. 75 76 We collect the next result job and pass it on to the build. 77 """ 78 while True: 79 result = self.builder.out_queue.get() 80 self.builder.ProcessResult(result) 81 self.builder.out_queue.task_done() 82 83 84class BuilderThread(threading.Thread): 85 """This thread builds U-Boot for a particular board. 86 87 An input queue provides each new job. We run 'make' to build U-Boot 88 and then pass the results on to the output queue. 89 90 Members: 91 builder: The builder which contains information we might need 92 thread_num: Our thread number (0-n-1), used to decide on a 93 temporary directory. If this is -1 then there are no threads 94 and we are the (only) main process 95 mrproper: Use 'make mrproper' before each reconfigure 96 per_board_out_dir: True to build in a separate persistent directory per 97 board rather than a thread-specific directory 98 test_exception: Used for testing; True to raise an exception instead of 99 reporting the build result 100 """ 101 def __init__(self, builder, thread_num, mrproper, per_board_out_dir, 102 test_exception=False): 103 """Set up a new builder thread""" 104 threading.Thread.__init__(self) 105 self.builder = builder 106 self.thread_num = thread_num 107 self.mrproper = mrproper 108 self.per_board_out_dir = per_board_out_dir 109 self.test_exception = test_exception 110 111 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 112 """Run 'make' on a particular commit and board. 113 114 The source code will already be checked out, so the 'commit' 115 argument is only for information. 116 117 Args: 118 commit: Commit object that is being built 119 brd: Board object that is being built 120 stage: Stage of the build. Valid stages are: 121 mrproper - can be called to clean source 122 config - called to configure for a board 123 build - the main make invocation - it does the build 124 args: A list of arguments to pass to 'make' 125 kwargs: A list of keyword arguments to pass to command.run_pipe() 126 127 Returns: 128 CommandResult object 129 """ 130 return self.builder.do_make(commit, brd, stage, cwd, *args, 131 **kwargs) 132 133 def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only, 134 force_build, force_build_failures, work_in_output, 135 adjust_cfg): 136 """Build a particular commit. 137 138 If the build is already done, and we are not forcing a build, we skip 139 the build and just return the previously-saved results. 140 141 Args: 142 commit_upto: Commit number to build (0...n-1) 143 brd: Board object to build 144 work_dir: Directory to which the source will be checked out 145 do_config: True to run a make <board>_defconfig on the source 146 config_only: Only configure the source, do not build it 147 force_build: Force a build even if one was previously done 148 force_build_failures: Force a bulid if the previous result showed 149 failure 150 work_in_output: Use the output directory as the work directory and 151 don't write to a separate output directory. 152 adjust_cfg (list of str): List of changes to make to .config file 153 before building. Each is one of (where C is either CONFIG_xxx 154 or just xxx): 155 C to enable C 156 ~C to disable C 157 C=val to set the value of C (val must have quotes if C is 158 a string Kconfig 159 160 Returns: 161 tuple containing: 162 - CommandResult object containing the results of the build 163 - boolean indicating whether 'make config' is still needed 164 """ 165 # Create a default result - it will be overwritte by the call to 166 # self.Make() below, in the event that we do a build. 167 result = command.CommandResult() 168 result.return_code = 0 169 if work_in_output or self.builder.in_tree: 170 out_dir = work_dir 171 else: 172 if self.per_board_out_dir: 173 out_rel_dir = os.path.join('..', brd.target) 174 else: 175 out_rel_dir = 'build' 176 out_dir = os.path.join(work_dir, out_rel_dir) 177 178 # Check if the job was already completed last time 179 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 180 result.already_done = os.path.exists(done_file) 181 will_build = (force_build or force_build_failures or 182 not result.already_done) 183 if result.already_done: 184 # Get the return code from that build and use it 185 with open(done_file, 'r') as fd: 186 try: 187 result.return_code = int(fd.readline()) 188 except ValueError: 189 # The file may be empty due to running out of disk space. 190 # Try a rebuild 191 result.return_code = RETURN_CODE_RETRY 192 193 # Check the signal that the build needs to be retried 194 if result.return_code == RETURN_CODE_RETRY: 195 will_build = True 196 elif will_build: 197 err_file = self.builder.GetErrFile(commit_upto, brd.target) 198 if os.path.exists(err_file) and os.stat(err_file).st_size: 199 result.stderr = 'bad' 200 elif not force_build: 201 # The build passed, so no need to build it again 202 will_build = False 203 204 if will_build: 205 # We are going to have to build it. First, get a toolchain 206 if not self.toolchain: 207 try: 208 self.toolchain = self.builder.toolchains.Select(brd.arch) 209 except ValueError as err: 210 result.return_code = 10 211 result.stdout = '' 212 result.stderr = str(err) 213 # TODO(sjg@chromium.org): This gets swallowed, but needs 214 # to be reported. 215 216 if self.toolchain: 217 # Checkout the right commit 218 if self.builder.commits: 219 commit = self.builder.commits[commit_upto] 220 if self.builder.checkout: 221 git_dir = os.path.join(work_dir, '.git') 222 gitutil.checkout(commit.hash, git_dir, work_dir, 223 force=True) 224 else: 225 commit = 'current' 226 227 # Set up the environment and command line 228 env = self.toolchain.MakeEnvironment(self.builder.full_path) 229 Mkdir(out_dir) 230 args = [] 231 cwd = work_dir 232 src_dir = os.path.realpath(work_dir) 233 if not self.builder.in_tree: 234 if commit_upto is None: 235 # In this case we are building in the original source 236 # directory (i.e. the current directory where buildman 237 # is invoked. The output directory is set to this 238 # thread's selected work directory. 239 # 240 # Symlinks can confuse U-Boot's Makefile since 241 # we may use '..' in our path, so remove them. 242 out_dir = os.path.realpath(out_dir) 243 args.append('O=%s' % out_dir) 244 cwd = None 245 src_dir = os.getcwd() 246 else: 247 args.append('O=%s' % out_rel_dir) 248 if self.builder.verbose_build: 249 args.append('V=1') 250 else: 251 args.append('-s') 252 if self.builder.num_jobs is not None: 253 args.extend(['-j', str(self.builder.num_jobs)]) 254 if self.builder.warnings_as_errors: 255 args.append('KCFLAGS=-Werror') 256 args.append('HOSTCFLAGS=-Werror') 257 if self.builder.allow_missing: 258 args.append('BINMAN_ALLOW_MISSING=1') 259 if self.builder.no_lto: 260 args.append('NO_LTO=1') 261 if self.builder.reproducible_builds: 262 args.append('SOURCE_DATE_EPOCH=0') 263 config_args = ['%s_defconfig' % brd.target] 264 config_out = '' 265 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 266 args.extend(self.toolchain.MakeArgs()) 267 268 # Remove any output targets. Since we use a build directory that 269 # was previously used by another board, it may have produced an 270 # SPL image. If we don't remove it (i.e. see do_config and 271 # self.mrproper below) then it will appear to be the output of 272 # this build, even if it does not produce SPL images. 273 build_dir = self.builder.GetBuildDir(commit_upto, brd.target) 274 for elf in BASE_ELF_FILENAMES: 275 fname = os.path.join(out_dir, elf) 276 if os.path.exists(fname): 277 os.remove(fname) 278 279 # If we need to reconfigure, do that now 280 cfg_file = os.path.join(out_dir, '.config') 281 cmd_list = [] 282 if do_config or adjust_cfg: 283 config_out = '' 284 if self.mrproper: 285 result = self.Make(commit, brd, 'mrproper', cwd, 286 'mrproper', *args, env=env) 287 config_out += result.combined 288 cmd_list.append([self.builder.gnu_make, 'mrproper', 289 *args]) 290 result = self.Make(commit, brd, 'config', cwd, 291 *(args + config_args), env=env) 292 cmd_list.append([self.builder.gnu_make] + args + 293 config_args) 294 config_out += result.combined 295 do_config = False # No need to configure next time 296 if adjust_cfg: 297 cfgutil.adjust_cfg_file(cfg_file, adjust_cfg) 298 if result.return_code == 0: 299 if config_only: 300 args.append('cfg') 301 result = self.Make(commit, brd, 'build', cwd, *args, 302 env=env) 303 cmd_list.append([self.builder.gnu_make] + args) 304 if (result.return_code == 2 and 305 ('Some images are invalid' in result.stderr)): 306 # This is handled later by the check for output in 307 # stderr 308 result.return_code = 0 309 if adjust_cfg: 310 errs = cfgutil.check_cfg_file(cfg_file, adjust_cfg) 311 if errs: 312 result.stderr += errs 313 result.return_code = 1 314 result.stderr = result.stderr.replace(src_dir + '/', '') 315 if self.builder.verbose_build: 316 result.stdout = config_out + result.stdout 317 result.cmd_list = cmd_list 318 else: 319 result.return_code = 1 320 result.stderr = 'No tool chain for %s\n' % brd.arch 321 result.already_done = False 322 323 result.toolchain = self.toolchain 324 result.brd = brd 325 result.commit_upto = commit_upto 326 result.out_dir = out_dir 327 return result, do_config 328 329 def _WriteResult(self, result, keep_outputs, work_in_output): 330 """Write a built result to the output directory. 331 332 Args: 333 result: CommandResult object containing result to write 334 keep_outputs: True to store the output binaries, False 335 to delete them 336 work_in_output: Use the output directory as the work directory and 337 don't write to a separate output directory. 338 """ 339 # If we think this might have been aborted with Ctrl-C, record the 340 # failure but not that we are 'done' with this board. A retry may fix 341 # it. 342 maybe_aborted = result.stderr and 'No child processes' in result.stderr 343 344 if result.return_code >= 0 and result.already_done: 345 return 346 347 # Write the output and stderr 348 output_dir = self.builder._GetOutputDir(result.commit_upto) 349 Mkdir(output_dir) 350 build_dir = self.builder.GetBuildDir(result.commit_upto, 351 result.brd.target) 352 Mkdir(build_dir) 353 354 outfile = os.path.join(build_dir, 'log') 355 with open(outfile, 'w') as fd: 356 if result.stdout: 357 fd.write(result.stdout) 358 359 errfile = self.builder.GetErrFile(result.commit_upto, 360 result.brd.target) 361 if result.stderr: 362 with open(errfile, 'w') as fd: 363 fd.write(result.stderr) 364 elif os.path.exists(errfile): 365 os.remove(errfile) 366 367 # Fatal error 368 if result.return_code < 0: 369 return 370 371 if result.toolchain: 372 # Write the build result and toolchain information. 373 done_file = self.builder.GetDoneFile(result.commit_upto, 374 result.brd.target) 375 with open(done_file, 'w') as fd: 376 if maybe_aborted: 377 # Special code to indicate we need to retry 378 fd.write('%s' % RETURN_CODE_RETRY) 379 else: 380 fd.write('%s' % result.return_code) 381 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 382 print('gcc', result.toolchain.gcc, file=fd) 383 print('path', result.toolchain.path, file=fd) 384 print('cross', result.toolchain.cross, file=fd) 385 print('arch', result.toolchain.arch, file=fd) 386 fd.write('%s' % result.return_code) 387 388 # Write out the image and function size information and an objdump 389 env = result.toolchain.MakeEnvironment(self.builder.full_path) 390 with open(os.path.join(build_dir, 'out-env'), 'wb') as fd: 391 for var in sorted(env.keys()): 392 fd.write(b'%s="%s"' % (var, env[var])) 393 394 with open(os.path.join(build_dir, 'out-cmd'), 'w', 395 encoding='utf-8') as fd: 396 for cmd in result.cmd_list: 397 print(' '.join(cmd), file=fd) 398 399 lines = [] 400 for fname in BASE_ELF_FILENAMES: 401 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 402 nm_result = command.run_pipe([cmd], capture=True, 403 capture_stderr=True, cwd=result.out_dir, 404 raise_on_error=False, env=env) 405 if nm_result.stdout: 406 nm = self.builder.GetFuncSizesFile(result.commit_upto, 407 result.brd.target, fname) 408 with open(nm, 'w') as fd: 409 print(nm_result.stdout, end=' ', file=fd) 410 411 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 412 dump_result = command.run_pipe([cmd], capture=True, 413 capture_stderr=True, cwd=result.out_dir, 414 raise_on_error=False, env=env) 415 rodata_size = '' 416 if dump_result.stdout: 417 objdump = self.builder.GetObjdumpFile(result.commit_upto, 418 result.brd.target, fname) 419 with open(objdump, 'w') as fd: 420 print(dump_result.stdout, end=' ', file=fd) 421 for line in dump_result.stdout.splitlines(): 422 fields = line.split() 423 if len(fields) > 5 and fields[1] == '.rodata': 424 rodata_size = fields[2] 425 426 cmd = ['%ssize' % self.toolchain.cross, fname] 427 size_result = command.run_pipe([cmd], capture=True, 428 capture_stderr=True, cwd=result.out_dir, 429 raise_on_error=False, env=env) 430 if size_result.stdout: 431 lines.append(size_result.stdout.splitlines()[1] + ' ' + 432 rodata_size) 433 434 # Extract the environment from U-Boot and dump it out 435 cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary', 436 '-j', '.rodata.default_environment', 437 'env/built-in.o', 'uboot.env'] 438 command.run_pipe([cmd], capture=True, 439 capture_stderr=True, cwd=result.out_dir, 440 raise_on_error=False, env=env) 441 ubootenv = os.path.join(result.out_dir, 'uboot.env') 442 if not work_in_output: 443 self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env']) 444 445 # Write out the image sizes file. This is similar to the output 446 # of binutil's 'size' utility, but it omits the header line and 447 # adds an additional hex value at the end of each line for the 448 # rodata size 449 if len(lines): 450 sizes = self.builder.GetSizesFile(result.commit_upto, 451 result.brd.target) 452 with open(sizes, 'w') as fd: 453 print('\n'.join(lines), file=fd) 454 455 if not work_in_output: 456 # Write out the configuration files, with a special case for SPL 457 for dirname in ['', 'spl', 'tpl']: 458 self.CopyFiles( 459 result.out_dir, build_dir, dirname, 460 ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', 461 '.config', 'include/autoconf.mk', 462 'include/generated/autoconf.h']) 463 464 # Now write the actual build output 465 if keep_outputs: 466 self.CopyFiles( 467 result.out_dir, build_dir, '', 468 ['u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL', 469 'include/autoconf.mk', 'spl/u-boot-spl*']) 470 471 def CopyFiles(self, out_dir, build_dir, dirname, patterns): 472 """Copy files from the build directory to the output. 473 474 Args: 475 out_dir: Path to output directory containing the files 476 build_dir: Place to copy the files 477 dirname: Source directory, '' for normal U-Boot, 'spl' for SPL 478 patterns: A list of filenames (strings) to copy, each relative 479 to the build directory 480 """ 481 for pattern in patterns: 482 file_list = glob.glob(os.path.join(out_dir, dirname, pattern)) 483 for fname in file_list: 484 target = os.path.basename(fname) 485 if dirname: 486 base, ext = os.path.splitext(target) 487 if ext: 488 target = '%s-%s%s' % (base, dirname, ext) 489 shutil.copy(fname, os.path.join(build_dir, target)) 490 491 def _SendResult(self, result): 492 """Send a result to the builder for processing 493 494 Args: 495 result: CommandResult object containing the results of the build 496 497 Raises: 498 ValueError if self.test_exception is true (for testing) 499 """ 500 if self.test_exception: 501 raise ValueError('test exception') 502 if self.thread_num != -1: 503 self.builder.out_queue.put(result) 504 else: 505 self.builder.ProcessResult(result) 506 507 def RunJob(self, job): 508 """Run a single job 509 510 A job consists of a building a list of commits for a particular board. 511 512 Args: 513 job: Job to build 514 515 Returns: 516 List of Result objects 517 """ 518 brd = job.brd 519 work_dir = self.builder.GetThreadDir(self.thread_num) 520 self.toolchain = None 521 if job.commits: 522 # Run 'make board_defconfig' on the first commit 523 do_config = True 524 commit_upto = 0 525 force_build = False 526 for commit_upto in range(0, len(job.commits), job.step): 527 result, request_config = self.RunCommit(commit_upto, brd, 528 work_dir, do_config, self.builder.config_only, 529 force_build or self.builder.force_build, 530 self.builder.force_build_failures, 531 job.work_in_output, job.adjust_cfg) 532 failed = result.return_code or result.stderr 533 did_config = do_config 534 if failed and not do_config: 535 # If our incremental build failed, try building again 536 # with a reconfig. 537 if self.builder.force_config_on_failure: 538 result, request_config = self.RunCommit(commit_upto, 539 brd, work_dir, True, False, True, False, 540 job.work_in_output, job.adjust_cfg) 541 did_config = True 542 if not self.builder.force_reconfig: 543 do_config = request_config 544 545 # If we built that commit, then config is done. But if we got 546 # an warning, reconfig next time to force it to build the same 547 # files that created warnings this time. Otherwise an 548 # incremental build may not build the same file, and we will 549 # think that the warning has gone away. 550 # We could avoid this by using -Werror everywhere... 551 # For errors, the problem doesn't happen, since presumably 552 # the build stopped and didn't generate output, so will retry 553 # that file next time. So we could detect warnings and deal 554 # with them specially here. For now, we just reconfigure if 555 # anything goes work. 556 # Of course this is substantially slower if there are build 557 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 558 # have problems). 559 if (failed and not result.already_done and not did_config and 560 self.builder.force_config_on_failure): 561 # If this build failed, try the next one with a 562 # reconfigure. 563 # Sometimes if the board_config.h file changes it can mess 564 # with dependencies, and we get: 565 # make: *** No rule to make target `include/autoconf.mk', 566 # needed by `depend'. 567 do_config = True 568 force_build = True 569 else: 570 force_build = False 571 if self.builder.force_config_on_failure: 572 if failed: 573 do_config = True 574 result.commit_upto = commit_upto 575 if result.return_code < 0: 576 raise ValueError('Interrupt') 577 578 # We have the build results, so output the result 579 self._WriteResult(result, job.keep_outputs, job.work_in_output) 580 self._SendResult(result) 581 else: 582 # Just build the currently checked-out build 583 result, request_config = self.RunCommit(None, brd, work_dir, True, 584 self.builder.config_only, True, 585 self.builder.force_build_failures, job.work_in_output, 586 job.adjust_cfg) 587 result.commit_upto = 0 588 self._WriteResult(result, job.keep_outputs, job.work_in_output) 589 self._SendResult(result) 590 591 def run(self): 592 """Our thread's run function 593 594 This thread picks a job from the queue, runs it, and then goes to the 595 next job. 596 """ 597 while True: 598 job = self.builder.queue.get() 599 try: 600 self.RunJob(job) 601 except Exception as e: 602 print('Thread exception (use -T0 to run without threads):', e) 603 self.builder.thread_exceptions.append(e) 604 self.builder.queue.task_done() 605