1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2012 The Chromium OS Authors. 3# 4 5from filelock import FileLock 6import os 7import shutil 8import sys 9import tempfile 10import time 11import unittest 12from unittest.mock import patch 13 14from buildman import board 15from buildman import boards 16from buildman import bsettings 17from buildman import builder 18from buildman import cfgutil 19from buildman import control 20from buildman import toolchain 21from patman import commit 22from u_boot_pylib import command 23from u_boot_pylib import terminal 24from u_boot_pylib import test_util 25from u_boot_pylib import tools 26 27use_network = True 28 29settings_data = ''' 30# Buildman settings file 31 32[toolchain] 33main: /usr/sbin 34 35[toolchain-alias] 36x86: i386 x86_64 37''' 38 39settings_data_wrapper = ''' 40# Buildman settings file 41 42[toolchain] 43main: /usr/sbin 44 45[toolchain-wrapper] 46wrapper = ccache 47''' 48 49settings_data_homedir = ''' 50# Buildman settings file 51 52[toolchain] 53main = ~/mypath 54 55[toolchain-prefix] 56x86 = ~/mypath-x86- 57''' 58 59migration = '''===================== WARNING ====================== 60This board does not use CONFIG_DM. CONFIG_DM will be 61compulsory starting with the v2020.01 release. 62Failure to update may result in board removal. 63See doc/develop/driver-model/migration.rst for more info. 64==================================================== 65''' 66 67errors = [ 68 '''main.c: In function 'main_loop': 69main.c:260:6: warning: unused variable 'joe' [-Wunused-variable] 70''', 71 '''main.c: In function 'main_loop2': 72main.c:295:2: error: 'fred' undeclared (first use in this function) 73main.c:295:2: note: each undeclared identifier is reported only once for each function it appears in 74make[1]: *** [main.o] Error 1 75make: *** [common/libcommon.o] Error 2 76Make failed 77''', 78 '''arch/arm/dts/socfpga_arria10_socdk_sdmmc.dtb: Warning \ 79(avoid_unnecessary_addr_size): /clocks: unnecessary #address-cells/#size-cells \ 80without "ranges" or child "reg" property 81''', 82 '''powerpc-linux-ld: warning: dot moved backwards before `.bss' 83powerpc-linux-ld: warning: dot moved backwards before `.bss' 84powerpc-linux-ld: u-boot: section .text lma 0xfffc0000 overlaps previous sections 85powerpc-linux-ld: u-boot: section .rodata lma 0xfffef3ec overlaps previous sections 86powerpc-linux-ld: u-boot: section .reloc lma 0xffffa400 overlaps previous sections 87powerpc-linux-ld: u-boot: section .data lma 0xffffcd38 overlaps previous sections 88powerpc-linux-ld: u-boot: section .u_boot_cmd lma 0xffffeb40 overlaps previous sections 89powerpc-linux-ld: u-boot: section .bootpg lma 0xfffff198 overlaps previous sections 90''', 91 '''In file included from %(basedir)sarch/sandbox/cpu/cpu.c:9:0: 92%(basedir)sarch/sandbox/include/asm/state.h:44:0: warning: "xxxx" redefined [enabled by default] 93%(basedir)sarch/sandbox/include/asm/state.h:43:0: note: this is the location of the previous definition 94%(basedir)sarch/sandbox/cpu/cpu.c: In function 'do_reset': 95%(basedir)sarch/sandbox/cpu/cpu.c:27:1: error: unknown type name 'blah' 96%(basedir)sarch/sandbox/cpu/cpu.c:28:12: error: expected declaration specifiers or '...' before numeric constant 97make[2]: *** [arch/sandbox/cpu/cpu.o] Error 1 98make[1]: *** [arch/sandbox/cpu] Error 2 99make[1]: *** Waiting for unfinished jobs.... 100In file included from %(basedir)scommon/board_f.c:55:0: 101%(basedir)sarch/sandbox/include/asm/state.h:44:0: warning: "xxxx" redefined [enabled by default] 102%(basedir)sarch/sandbox/include/asm/state.h:43:0: note: this is the location of the previous definition 103make: *** [sub-make] Error 2 104''' 105] 106 107 108# hash, subject, return code, list of errors/warnings 109commits = [ 110 ['1234', 'upstream/master, migration warning', 0, []], 111 ['5678', 'Second commit, a warning', 0, errors[0:1]], 112 ['9012', 'Third commit, error', 1, errors[0:2]], 113 ['3456', 'Fourth commit, warning', 0, [errors[0], errors[2]]], 114 ['7890', 'Fifth commit, link errors', 1, [errors[0], errors[3]]], 115 ['abcd', 'Sixth commit, fixes all errors', 0, []], 116 ['ef01', 'Seventh commit, fix migration, check directory suppression', 1, 117 [errors[4]]], 118] 119 120BOARDS = [ 121 ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0', ''], 122 ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''], 123 ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''], 124 ['Active', 'powerpc', 'mpc83xx', '', 'Tester', 'PowerPC board 2', 'board3', ''], 125 ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''], 126] 127 128BASE_DIR = 'base' 129 130OUTCOME_OK, OUTCOME_WARN, OUTCOME_ERR = range(3) 131 132class Options: 133 """Class that holds build options""" 134 pass 135 136class TestBuild(unittest.TestCase): 137 """Test buildman 138 139 TODO: Write tests for the rest of the functionality 140 """ 141 def setUp(self): 142 # Set up commits to build 143 self.commits = [] 144 sequence = 0 145 for commit_info in commits: 146 comm = commit.Commit(commit_info[0]) 147 comm.subject = commit_info[1] 148 comm.return_code = commit_info[2] 149 comm.error_list = commit_info[3] 150 if sequence < 6: 151 comm.error_list += [migration] 152 comm.sequence = sequence 153 sequence += 1 154 self.commits.append(comm) 155 156 # Set up boards to build 157 self.brds = boards.Boards() 158 for brd in BOARDS: 159 self.brds.add_board(board.Board(*brd)) 160 self.brds.select_boards([]) 161 162 # Add some test settings 163 bsettings.setup(None) 164 bsettings.add_file(settings_data) 165 166 # Set up the toolchains 167 self.toolchains = toolchain.Toolchains() 168 self.toolchains.Add('arm-linux-gcc', test=False) 169 self.toolchains.Add('sparc-linux-gcc', test=False) 170 self.toolchains.Add('powerpc-linux-gcc', test=False) 171 self.toolchains.Add('/path/to/aarch64-linux-gcc', test=False) 172 self.toolchains.Add('gcc', test=False) 173 174 # Avoid sending any output 175 terminal.set_print_test_mode() 176 self._col = terminal.Color() 177 178 self.base_dir = tempfile.mkdtemp() 179 if not os.path.isdir(self.base_dir): 180 os.mkdir(self.base_dir) 181 182 self.cur_time = 0 183 self.valid_pids = [] 184 self.finish_time = None 185 self.finish_pid = None 186 187 def tearDown(self): 188 shutil.rmtree(self.base_dir) 189 190 def Make(self, commit, brd, stage, *args, **kwargs): 191 result = command.CommandResult() 192 boardnum = int(brd.target[-1]) 193 result.return_code = 0 194 result.stderr = '' 195 result.stdout = ('This is the test output for board %s, commit %s' % 196 (brd.target, commit.hash)) 197 if ((boardnum >= 1 and boardnum >= commit.sequence) or 198 boardnum == 4 and commit.sequence == 6): 199 result.return_code = commit.return_code 200 result.stderr = (''.join(commit.error_list) 201 % {'basedir' : self.base_dir + '/.bm-work/00/'}) 202 elif commit.sequence < 6: 203 result.stderr = migration 204 205 result.combined = result.stdout + result.stderr 206 return result 207 208 def assertSummary(self, text, arch, plus, brds, outcome=OUTCOME_ERR): 209 col = self._col 210 expected_colour = (col.GREEN if outcome == OUTCOME_OK else 211 col.YELLOW if outcome == OUTCOME_WARN else col.RED) 212 expect = '%10s: ' % arch 213 # TODO(sjg@chromium.org): If plus is '', we shouldn't need this 214 expect += ' ' + col.build(expected_colour, plus) 215 expect += ' ' 216 for brd in brds: 217 expect += col.build(expected_colour, ' %s' % brd) 218 self.assertEqual(text, expect) 219 220 def _SetupTest(self, echo_lines=False, threads=1, **kwdisplay_args): 221 """Set up the test by running a build and summary 222 223 Args: 224 echo_lines: True to echo lines to the terminal to aid test 225 development 226 kwdisplay_args: Dict of arguments to pass to 227 Builder.SetDisplayOptions() 228 229 Returns: 230 Iterator containing the output lines, each a PrintLine() object 231 """ 232 build = builder.Builder(self.toolchains, self.base_dir, None, threads, 233 2, checkout=False, show_unknown=False) 234 build.do_make = self.Make 235 board_selected = self.brds.get_selected_dict() 236 237 # Build the boards for the pre-defined commits and warnings/errors 238 # associated with each. This calls our Make() to inject the fake output. 239 build.build_boards(self.commits, board_selected, keep_outputs=False, 240 verbose=False) 241 lines = terminal.get_print_test_lines() 242 count = 0 243 for line in lines: 244 if line.text.strip(): 245 count += 1 246 247 # We should get two starting messages, an update for every commit built 248 # and a summary message 249 self.assertEqual(count, len(commits) * len(BOARDS) + 3) 250 build.set_display_options(**kwdisplay_args); 251 build.show_summary(self.commits, board_selected) 252 if echo_lines: 253 terminal.echo_print_test_lines() 254 return iter(terminal.get_print_test_lines()) 255 256 def _CheckOutput(self, lines, list_error_boards=False, 257 filter_dtb_warnings=False, 258 filter_migration_warnings=False): 259 """Check for expected output from the build summary 260 261 Args: 262 lines: Iterator containing the lines returned from the summary 263 list_error_boards: Adjust the check for output produced with the 264 --list-error-boards flag 265 filter_dtb_warnings: Adjust the check for output produced with the 266 --filter-dtb-warnings flag 267 """ 268 def add_line_prefix(prefix, brds, error_str, colour): 269 """Add a prefix to each line of a string 270 271 The training \n in error_str is removed before processing 272 273 Args: 274 prefix: String prefix to add 275 error_str: Error string containing the lines 276 colour: Expected colour for the line. Note that the board list, 277 if present, always appears in magenta 278 279 Returns: 280 New string where each line has the prefix added 281 """ 282 lines = error_str.strip().splitlines() 283 new_lines = [] 284 for line in lines: 285 if brds: 286 expect = self._col.build(colour, prefix + '(') 287 expect += self._col.build(self._col.MAGENTA, brds, 288 bright=False) 289 expect += self._col.build(colour, ') %s' % line) 290 else: 291 expect = self._col.build(colour, prefix + line) 292 new_lines.append(expect) 293 return '\n'.join(new_lines) 294 295 col = terminal.Color() 296 boards01234 = ('board0 board1 board2 board3 board4' 297 if list_error_boards else '') 298 boards1234 = 'board1 board2 board3 board4' if list_error_boards else '' 299 boards234 = 'board2 board3 board4' if list_error_boards else '' 300 boards34 = 'board3 board4' if list_error_boards else '' 301 boards4 = 'board4' if list_error_boards else '' 302 303 # Upstream commit: migration warnings only 304 self.assertEqual(next(lines).text, '01: %s' % commits[0][1]) 305 306 if not filter_migration_warnings: 307 self.assertSummary(next(lines).text, 'arm', 'w+', 308 ['board0', 'board1'], outcome=OUTCOME_WARN) 309 self.assertSummary(next(lines).text, 'powerpc', 'w+', 310 ['board2', 'board3'], outcome=OUTCOME_WARN) 311 self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], 312 outcome=OUTCOME_WARN) 313 314 self.assertEqual(next(lines).text, 315 add_line_prefix('+', boards01234, migration, col.RED)) 316 317 # Second commit: all archs should fail with warnings 318 self.assertEqual(next(lines).text, '02: %s' % commits[1][1]) 319 320 if filter_migration_warnings: 321 self.assertSummary(next(lines).text, 'arm', 'w+', 322 ['board1'], outcome=OUTCOME_WARN) 323 self.assertSummary(next(lines).text, 'powerpc', 'w+', 324 ['board2', 'board3'], outcome=OUTCOME_WARN) 325 self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], 326 outcome=OUTCOME_WARN) 327 328 # Second commit: The warnings should be listed 329 self.assertEqual(next(lines).text, 330 add_line_prefix('w+', boards1234, errors[0], col.YELLOW)) 331 332 # Third commit: Still fails 333 self.assertEqual(next(lines).text, '03: %s' % commits[2][1]) 334 if filter_migration_warnings: 335 self.assertSummary(next(lines).text, 'arm', '', 336 ['board1'], outcome=OUTCOME_OK) 337 self.assertSummary(next(lines).text, 'powerpc', '+', 338 ['board2', 'board3']) 339 self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) 340 341 # Expect a compiler error 342 self.assertEqual(next(lines).text, 343 add_line_prefix('+', boards234, errors[1], col.RED)) 344 345 # Fourth commit: Compile errors are fixed, just have warning for board3 346 self.assertEqual(next(lines).text, '04: %s' % commits[3][1]) 347 if filter_migration_warnings: 348 expect = '%10s: ' % 'powerpc' 349 expect += ' ' + col.build(col.GREEN, '') 350 expect += ' ' 351 expect += col.build(col.GREEN, ' %s' % 'board2') 352 expect += ' ' + col.build(col.YELLOW, 'w+') 353 expect += ' ' 354 expect += col.build(col.YELLOW, ' %s' % 'board3') 355 self.assertEqual(next(lines).text, expect) 356 else: 357 self.assertSummary(next(lines).text, 'powerpc', 'w+', 358 ['board2', 'board3'], outcome=OUTCOME_WARN) 359 self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], 360 outcome=OUTCOME_WARN) 361 362 # Compile error fixed 363 self.assertEqual(next(lines).text, 364 add_line_prefix('-', boards234, errors[1], col.GREEN)) 365 366 if not filter_dtb_warnings: 367 self.assertEqual( 368 next(lines).text, 369 add_line_prefix('w+', boards34, errors[2], col.YELLOW)) 370 371 # Fifth commit 372 self.assertEqual(next(lines).text, '05: %s' % commits[4][1]) 373 if filter_migration_warnings: 374 self.assertSummary(next(lines).text, 'powerpc', '', ['board3'], 375 outcome=OUTCOME_OK) 376 self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) 377 378 # The second line of errors[3] is a duplicate, so buildman will drop it 379 expect = errors[3].rstrip().split('\n') 380 expect = [expect[0]] + expect[2:] 381 expect = '\n'.join(expect) 382 self.assertEqual(next(lines).text, 383 add_line_prefix('+', boards4, expect, col.RED)) 384 385 if not filter_dtb_warnings: 386 self.assertEqual( 387 next(lines).text, 388 add_line_prefix('w-', boards34, errors[2], col.CYAN)) 389 390 # Sixth commit 391 self.assertEqual(next(lines).text, '06: %s' % commits[5][1]) 392 if filter_migration_warnings: 393 self.assertSummary(next(lines).text, 'sandbox', '', ['board4'], 394 outcome=OUTCOME_OK) 395 else: 396 self.assertSummary(next(lines).text, 'sandbox', 'w+', ['board4'], 397 outcome=OUTCOME_WARN) 398 399 # The second line of errors[3] is a duplicate, so buildman will drop it 400 expect = errors[3].rstrip().split('\n') 401 expect = [expect[0]] + expect[2:] 402 expect = '\n'.join(expect) 403 self.assertEqual(next(lines).text, 404 add_line_prefix('-', boards4, expect, col.GREEN)) 405 self.assertEqual(next(lines).text, 406 add_line_prefix('w-', boards4, errors[0], col.CYAN)) 407 408 # Seventh commit 409 self.assertEqual(next(lines).text, '07: %s' % commits[6][1]) 410 if filter_migration_warnings: 411 self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) 412 else: 413 self.assertSummary(next(lines).text, 'arm', '', ['board0', 'board1'], 414 outcome=OUTCOME_OK) 415 self.assertSummary(next(lines).text, 'powerpc', '', 416 ['board2', 'board3'], outcome=OUTCOME_OK) 417 self.assertSummary(next(lines).text, 'sandbox', '+', ['board4']) 418 419 # Pick out the correct error lines 420 expect_str = errors[4].rstrip().replace('%(basedir)s', '').split('\n') 421 expect = expect_str[3:8] + [expect_str[-1]] 422 expect = '\n'.join(expect) 423 if not filter_migration_warnings: 424 self.assertEqual( 425 next(lines).text, 426 add_line_prefix('-', boards01234, migration, col.GREEN)) 427 428 self.assertEqual(next(lines).text, 429 add_line_prefix('+', boards4, expect, col.RED)) 430 431 # Now the warnings lines 432 expect = [expect_str[0]] + expect_str[10:12] + [expect_str[9]] 433 expect = '\n'.join(expect) 434 self.assertEqual(next(lines).text, 435 add_line_prefix('w+', boards4, expect, col.YELLOW)) 436 437 def testOutput(self): 438 """Test basic builder operation and output 439 440 This does a line-by-line verification of the summary output. 441 """ 442 lines = self._SetupTest(show_errors=True) 443 self._CheckOutput(lines, list_error_boards=False, 444 filter_dtb_warnings=False) 445 446 def testErrorBoards(self): 447 """Test output with --list-error-boards 448 449 This does a line-by-line verification of the summary output. 450 """ 451 lines = self._SetupTest(show_errors=True, list_error_boards=True) 452 self._CheckOutput(lines, list_error_boards=True) 453 454 def testFilterDtb(self): 455 """Test output with --filter-dtb-warnings 456 457 This does a line-by-line verification of the summary output. 458 """ 459 lines = self._SetupTest(show_errors=True, filter_dtb_warnings=True) 460 self._CheckOutput(lines, filter_dtb_warnings=True) 461 462 def testFilterMigration(self): 463 """Test output with --filter-migration-warnings 464 465 This does a line-by-line verification of the summary output. 466 """ 467 lines = self._SetupTest(show_errors=True, 468 filter_migration_warnings=True) 469 self._CheckOutput(lines, filter_migration_warnings=True) 470 471 def testSingleThread(self): 472 """Test operation without threading""" 473 lines = self._SetupTest(show_errors=True, threads=0) 474 self._CheckOutput(lines, list_error_boards=False, 475 filter_dtb_warnings=False) 476 477 def _testGit(self): 478 """Test basic builder operation by building a branch""" 479 options = Options() 480 options.git = os.getcwd() 481 options.summary = False 482 options.jobs = None 483 options.dry_run = False 484 #options.git = os.path.join(self.base_dir, 'repo') 485 options.branch = 'test-buildman' 486 options.force_build = False 487 options.list_tool_chains = False 488 options.count = -1 489 options.git_dir = None 490 options.threads = None 491 options.show_unknown = False 492 options.quick = False 493 options.show_errors = False 494 options.keep_outputs = False 495 args = ['tegra20'] 496 control.do_buildman(options, args) 497 498 def testBoardSingle(self): 499 """Test single board selection""" 500 self.assertEqual(self.brds.select_boards(['sandbox']), 501 ({'all': ['board4'], 'sandbox': ['board4']}, [])) 502 503 def testBoardArch(self): 504 """Test single board selection""" 505 self.assertEqual(self.brds.select_boards(['arm']), 506 ({'all': ['board0', 'board1'], 507 'arm': ['board0', 'board1']}, [])) 508 509 def testBoardArchSingle(self): 510 """Test single board selection""" 511 self.assertEqual(self.brds.select_boards(['arm sandbox']), 512 ({'sandbox': ['board4'], 513 'all': ['board0', 'board1', 'board4'], 514 'arm': ['board0', 'board1']}, [])) 515 516 517 def testBoardArchSingleMultiWord(self): 518 """Test single board selection""" 519 self.assertEqual(self.brds.select_boards(['arm', 'sandbox']), 520 ({'sandbox': ['board4'], 521 'all': ['board0', 'board1', 'board4'], 522 'arm': ['board0', 'board1']}, [])) 523 524 def testBoardSingleAnd(self): 525 """Test single board selection""" 526 self.assertEqual(self.brds.select_boards(['Tester & arm']), 527 ({'Tester&arm': ['board0', 'board1'], 528 'all': ['board0', 'board1']}, [])) 529 530 def testBoardTwoAnd(self): 531 """Test single board selection""" 532 self.assertEqual(self.brds.select_boards(['Tester', '&', 'arm', 533 'Tester' '&', 'powerpc', 534 'sandbox']), 535 ({'sandbox': ['board4'], 536 'all': ['board0', 'board1', 'board2', 'board3', 537 'board4'], 538 'Tester&powerpc': ['board2', 'board3'], 539 'Tester&arm': ['board0', 'board1']}, [])) 540 541 def testBoardAll(self): 542 """Test single board selection""" 543 self.assertEqual(self.brds.select_boards([]), 544 ({'all': ['board0', 'board1', 'board2', 'board3', 545 'board4']}, [])) 546 547 def testBoardRegularExpression(self): 548 """Test single board selection""" 549 self.assertEqual(self.brds.select_boards(['T.*r&^Po']), 550 ({'all': ['board2', 'board3'], 551 'T.*r&^Po': ['board2', 'board3']}, [])) 552 553 def testBoardDuplicate(self): 554 """Test single board selection""" 555 self.assertEqual(self.brds.select_boards(['sandbox sandbox', 556 'sandbox']), 557 ({'all': ['board4'], 'sandbox': ['board4']}, [])) 558 def CheckDirs(self, build, dirname): 559 self.assertEqual('base%s' % dirname, build.get_output_dir(1)) 560 self.assertEqual('base%s/fred' % dirname, 561 build.get_build_dir(1, 'fred')) 562 self.assertEqual('base%s/fred/done' % dirname, 563 build.get_done_file(1, 'fred')) 564 self.assertEqual('base%s/fred/u-boot.sizes' % dirname, 565 build.get_func_sizes_file(1, 'fred', 'u-boot')) 566 self.assertEqual('base%s/fred/u-boot.objdump' % dirname, 567 build.get_objdump_file(1, 'fred', 'u-boot')) 568 self.assertEqual('base%s/fred/err' % dirname, 569 build.get_err_file(1, 'fred')) 570 571 def testOutputDir(self): 572 build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, 573 checkout=False, show_unknown=False) 574 build.commits = self.commits 575 build.commit_count = len(self.commits) 576 subject = self.commits[1].subject.translate(builder.trans_valid_chars) 577 dirname ='/%02d_g%s_%s' % (2, commits[1][0], subject[:20]) 578 self.CheckDirs(build, dirname) 579 580 def testOutputDirCurrent(self): 581 build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, 582 checkout=False, show_unknown=False) 583 build.commits = None 584 build.commit_count = 0 585 self.CheckDirs(build, '/current') 586 587 def testOutputDirNoSubdirs(self): 588 build = builder.Builder(self.toolchains, BASE_DIR, None, 1, 2, 589 checkout=False, show_unknown=False, 590 no_subdirs=True) 591 build.commits = None 592 build.commit_count = 0 593 self.CheckDirs(build, '') 594 595 def testToolchainAliases(self): 596 self.assertTrue(self.toolchains.Select('arm') != None) 597 with self.assertRaises(ValueError): 598 self.toolchains.Select('no-arch') 599 with self.assertRaises(ValueError): 600 self.toolchains.Select('x86') 601 602 self.toolchains = toolchain.Toolchains() 603 self.toolchains.Add('x86_64-linux-gcc', test=False) 604 self.assertTrue(self.toolchains.Select('x86') != None) 605 606 self.toolchains = toolchain.Toolchains() 607 self.toolchains.Add('i386-linux-gcc', test=False) 608 self.assertTrue(self.toolchains.Select('x86') != None) 609 610 def testToolchainDownload(self): 611 """Test that we can download toolchains""" 612 if use_network: 613 with terminal.capture() as (stdout, stderr): 614 url = self.toolchains.LocateArchUrl('arm') 615 self.assertRegex(url, 'https://www.kernel.org/pub/tools/' 616 'crosstool/files/bin/x86_64/.*/' 617 'x86_64-gcc-.*-nolibc[-_]arm-.*linux-gnueabi.tar.xz') 618 619 def testGetEnvArgs(self): 620 """Test the GetEnvArgs() function""" 621 tc = self.toolchains.Select('arm') 622 self.assertEqual('arm-linux-', 623 tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) 624 self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_PATH)) 625 self.assertEqual('arm', 626 tc.GetEnvArgs(toolchain.VAR_ARCH)) 627 self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) 628 629 tc = self.toolchains.Select('sandbox') 630 self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) 631 632 self.toolchains.Add('/path/to/x86_64-linux-gcc', test=False) 633 tc = self.toolchains.Select('x86') 634 self.assertEqual('/path/to', 635 tc.GetEnvArgs(toolchain.VAR_PATH)) 636 tc.override_toolchain = 'clang' 637 self.assertEqual('HOSTCC=clang CC=clang', 638 tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) 639 640 # Test config with ccache wrapper 641 bsettings.setup(None) 642 bsettings.add_file(settings_data_wrapper) 643 644 tc = self.toolchains.Select('arm') 645 self.assertEqual('ccache arm-linux-', 646 tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) 647 648 tc = self.toolchains.Select('sandbox') 649 self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) 650 651 def testMakeEnvironment(self): 652 """Test the MakeEnvironment function""" 653 tc = self.toolchains.Select('arm') 654 env = tc.MakeEnvironment(False) 655 self.assertEqual(env[b'CROSS_COMPILE'], b'arm-linux-') 656 657 tc = self.toolchains.Select('sandbox') 658 env = tc.MakeEnvironment(False) 659 self.assertTrue(b'CROSS_COMPILE' not in env) 660 661 # Test config with ccache wrapper 662 bsettings.setup(None) 663 bsettings.add_file(settings_data_wrapper) 664 665 tc = self.toolchains.Select('arm') 666 env = tc.MakeEnvironment(False) 667 self.assertEqual(env[b'CROSS_COMPILE'], b'ccache arm-linux-') 668 669 tc = self.toolchains.Select('sandbox') 670 env = tc.MakeEnvironment(False) 671 self.assertTrue(b'CROSS_COMPILE' not in env) 672 673 def testPrepareOutputSpace(self): 674 def _Touch(fname): 675 tools.write_file(os.path.join(base_dir, fname), b'') 676 677 base_dir = tempfile.mkdtemp() 678 679 # Add various files that we want removed and left alone 680 to_remove = ['01_g0982734987_title', '102_g92bf_title', 681 '01_g2938abd8_title'] 682 to_leave = ['something_else', '01-something.patch', '01_another'] 683 for name in to_remove + to_leave: 684 _Touch(name) 685 686 build = builder.Builder(self.toolchains, base_dir, None, 1, 2) 687 build.commits = self.commits 688 build.commit_count = len(commits) 689 result = set(build._get_output_space_removals()) 690 expected = set([os.path.join(base_dir, f) for f in to_remove]) 691 self.assertEqual(expected, result) 692 693 def test_adjust_cfg_nop(self): 694 """check various adjustments of config that are nops""" 695 # enable an enabled CONFIG 696 self.assertEqual( 697 'CONFIG_FRED=y', 698 cfgutil.adjust_cfg_line('CONFIG_FRED=y', {'FRED':'FRED'})[0]) 699 700 # disable a disabled CONFIG 701 self.assertEqual( 702 '# CONFIG_FRED is not set', 703 cfgutil.adjust_cfg_line( 704 '# CONFIG_FRED is not set', {'FRED':'~FRED'})[0]) 705 706 # use the adjust_cfg_lines() function 707 self.assertEqual( 708 ['CONFIG_FRED=y'], 709 cfgutil.adjust_cfg_lines(['CONFIG_FRED=y'], {'FRED':'FRED'})) 710 self.assertEqual( 711 ['# CONFIG_FRED is not set'], 712 cfgutil.adjust_cfg_lines(['CONFIG_FRED=y'], {'FRED':'~FRED'})) 713 714 # handling an empty line 715 self.assertEqual('#', cfgutil.adjust_cfg_line('#', {'FRED':'~FRED'})[0]) 716 717 def test_adjust_cfg(self): 718 """check various adjustments of config""" 719 # disable a CONFIG 720 self.assertEqual( 721 '# CONFIG_FRED is not set', 722 cfgutil.adjust_cfg_line('CONFIG_FRED=1' , {'FRED':'~FRED'})[0]) 723 724 # enable a disabled CONFIG 725 self.assertEqual( 726 'CONFIG_FRED=y', 727 cfgutil.adjust_cfg_line( 728 '# CONFIG_FRED is not set', {'FRED':'FRED'})[0]) 729 730 # enable a CONFIG that doesn't exist 731 self.assertEqual( 732 ['CONFIG_FRED=y'], 733 cfgutil.adjust_cfg_lines([], {'FRED':'FRED'})) 734 735 # disable a CONFIG that doesn't exist 736 self.assertEqual( 737 ['# CONFIG_FRED is not set'], 738 cfgutil.adjust_cfg_lines([], {'FRED':'~FRED'})) 739 740 # disable a value CONFIG 741 self.assertEqual( 742 '# CONFIG_FRED is not set', 743 cfgutil.adjust_cfg_line('CONFIG_FRED="fred"' , {'FRED':'~FRED'})[0]) 744 745 # setting a value CONFIG 746 self.assertEqual( 747 'CONFIG_FRED="fred"', 748 cfgutil.adjust_cfg_line('# CONFIG_FRED is not set' , 749 {'FRED':'FRED="fred"'})[0]) 750 751 # changing a value CONFIG 752 self.assertEqual( 753 'CONFIG_FRED="fred"', 754 cfgutil.adjust_cfg_line('CONFIG_FRED="ernie"' , 755 {'FRED':'FRED="fred"'})[0]) 756 757 # setting a value for a CONFIG that doesn't exist 758 self.assertEqual( 759 ['CONFIG_FRED="fred"'], 760 cfgutil.adjust_cfg_lines([], {'FRED':'FRED="fred"'})) 761 762 def test_convert_adjust_cfg_list(self): 763 """Check conversion of the list of changes into a dict""" 764 self.assertEqual({}, cfgutil.convert_list_to_dict(None)) 765 766 expect = { 767 'FRED':'FRED', 768 'MARY':'~MARY', 769 'JOHN':'JOHN=0x123', 770 'ALICE':'ALICE="alice"', 771 'AMY':'AMY', 772 'ABE':'~ABE', 773 'MARK':'MARK=0x456', 774 'ANNA':'ANNA="anna"', 775 } 776 actual = cfgutil.convert_list_to_dict( 777 ['FRED', '~MARY', 'JOHN=0x123', 'ALICE="alice"', 778 'CONFIG_AMY', '~CONFIG_ABE', 'CONFIG_MARK=0x456', 779 'CONFIG_ANNA="anna"']) 780 self.assertEqual(expect, actual) 781 782 def test_check_cfg_file(self): 783 """Test check_cfg_file detects conflicts as expected""" 784 # Check failure to disable CONFIG 785 result = cfgutil.check_cfg_lines(['CONFIG_FRED=1'], {'FRED':'~FRED'}) 786 self.assertEqual([['~FRED', 'CONFIG_FRED=1']], result) 787 788 result = cfgutil.check_cfg_lines( 789 ['CONFIG_FRED=1', 'CONFIG_MARY="mary"'], {'FRED':'~FRED'}) 790 self.assertEqual([['~FRED', 'CONFIG_FRED=1']], result) 791 792 result = cfgutil.check_cfg_lines( 793 ['CONFIG_FRED=1', 'CONFIG_MARY="mary"'], {'MARY':'~MARY'}) 794 self.assertEqual([['~MARY', 'CONFIG_MARY="mary"']], result) 795 796 # Check failure to enable CONFIG 797 result = cfgutil.check_cfg_lines( 798 ['# CONFIG_FRED is not set'], {'FRED':'FRED'}) 799 self.assertEqual([['FRED', '# CONFIG_FRED is not set']], result) 800 801 # Check failure to set CONFIG value 802 result = cfgutil.check_cfg_lines( 803 ['# CONFIG_FRED is not set', 'CONFIG_MARY="not"'], 804 {'MARY':'MARY="mary"', 'FRED':'FRED'}) 805 self.assertEqual([ 806 ['FRED', '# CONFIG_FRED is not set'], 807 ['MARY="mary"', 'CONFIG_MARY="not"']], result) 808 809 # Check failure to add CONFIG value 810 result = cfgutil.check_cfg_lines([], {'MARY':'MARY="mary"'}) 811 self.assertEqual([ 812 ['MARY="mary"', 'Missing expected line: CONFIG_MARY="mary"']], result) 813 814 def get_procs(self): 815 running_fname = os.path.join(self.base_dir, control.RUNNING_FNAME) 816 items = tools.read_file(running_fname, binary=False).split() 817 return [int(x) for x in items] 818 819 def get_time(self): 820 return self.cur_time 821 822 def inc_time(self, amount): 823 self.cur_time += amount 824 825 # Handle a process exiting 826 if self.finish_time == self.cur_time: 827 self.valid_pids = [pid for pid in self.valid_pids 828 if pid != self.finish_pid] 829 830 def kill(self, pid, signal): 831 if pid not in self.valid_pids: 832 raise OSError('Invalid PID') 833 834 def test_process_limit(self): 835 """Test wait_for_process_limit() function""" 836 tmpdir = self.base_dir 837 838 with (patch('time.time', side_effect=self.get_time), 839 patch('time.perf_counter', side_effect=self.get_time), 840 patch('time.monotonic', side_effect=self.get_time), 841 patch('time.sleep', side_effect=self.inc_time), 842 patch('os.kill', side_effect=self.kill)): 843 # Grab the process. Since there is no other profcess, this should 844 # immediately succeed 845 control.wait_for_process_limit(1, tmpdir=tmpdir, pid=1) 846 lines = terminal.get_print_test_lines() 847 self.assertEqual(0, self.cur_time) 848 self.assertEqual('Waiting for other buildman processes...', 849 lines[0].text) 850 self.assertEqual(self._col.RED, lines[0].colour) 851 self.assertEqual(False, lines[0].newline) 852 self.assertEqual(True, lines[0].bright) 853 854 self.assertEqual('done...', lines[1].text) 855 self.assertEqual(None, lines[1].colour) 856 self.assertEqual(False, lines[1].newline) 857 self.assertEqual(True, lines[1].bright) 858 859 self.assertEqual('starting build', lines[2].text) 860 self.assertEqual([1], control.read_procs(tmpdir)) 861 self.assertEqual(None, lines[2].colour) 862 self.assertEqual(False, lines[2].newline) 863 self.assertEqual(True, lines[2].bright) 864 865 # Try again, with a different PID...this should eventually timeout 866 # and start the build anyway 867 self.cur_time = 0 868 self.valid_pids = [1] 869 control.wait_for_process_limit(1, tmpdir=tmpdir, pid=2) 870 lines = terminal.get_print_test_lines() 871 self.assertEqual('Waiting for other buildman processes...', 872 lines[0].text) 873 self.assertEqual('timeout...', lines[1].text) 874 self.assertEqual(None, lines[1].colour) 875 self.assertEqual(False, lines[1].newline) 876 self.assertEqual(True, lines[1].bright) 877 self.assertEqual('starting build', lines[2].text) 878 self.assertEqual([1, 2], control.read_procs(tmpdir)) 879 self.assertEqual(control.RUN_WAIT_S, self.cur_time) 880 881 # Check lock-busting 882 self.cur_time = 0 883 self.valid_pids = [1, 2] 884 lock_fname = os.path.join(tmpdir, control.LOCK_FNAME) 885 lock = FileLock(lock_fname) 886 lock.acquire(timeout=1) 887 control.wait_for_process_limit(1, tmpdir=tmpdir, pid=3) 888 lines = terminal.get_print_test_lines() 889 self.assertEqual('Waiting for other buildman processes...', 890 lines[0].text) 891 self.assertEqual('failed to get lock: busting...', lines[1].text) 892 self.assertEqual(None, lines[1].colour) 893 self.assertEqual(False, lines[1].newline) 894 self.assertEqual(True, lines[1].bright) 895 self.assertEqual('timeout...', lines[2].text) 896 self.assertEqual('starting build', lines[3].text) 897 self.assertEqual([1, 2, 3], control.read_procs(tmpdir)) 898 self.assertEqual(control.RUN_WAIT_S, self.cur_time) 899 lock.release() 900 901 # Check handling of dead processes. Here we have PID 2 as a running 902 # process, even though the PID file contains 1, 2 and 3. So we can 903 # add one more PID, to make 2 and 4 904 self.cur_time = 0 905 self.valid_pids = [2] 906 control.wait_for_process_limit(2, tmpdir=tmpdir, pid=4) 907 lines = terminal.get_print_test_lines() 908 self.assertEqual('Waiting for other buildman processes...', 909 lines[0].text) 910 self.assertEqual('done...', lines[1].text) 911 self.assertEqual('starting build', lines[2].text) 912 self.assertEqual([2, 4], control.read_procs(tmpdir)) 913 self.assertEqual(0, self.cur_time) 914 915 # Try again, with PID 2 quitting at time 50. This allows the new 916 # build to start 917 self.cur_time = 0 918 self.valid_pids = [2, 4] 919 self.finish_pid = 2 920 self.finish_time = 50 921 control.wait_for_process_limit(2, tmpdir=tmpdir, pid=5) 922 lines = terminal.get_print_test_lines() 923 self.assertEqual('Waiting for other buildman processes...', 924 lines[0].text) 925 self.assertEqual('done...', lines[1].text) 926 self.assertEqual('starting build', lines[2].text) 927 self.assertEqual([4, 5], control.read_procs(tmpdir)) 928 self.assertEqual(self.finish_time, self.cur_time) 929 930 def call_make_environment(self, tchn, full_path, in_env=None): 931 """Call Toolchain.MakeEnvironment() and process the result 932 933 Args: 934 tchn (Toolchain): Toolchain to use 935 full_path (bool): True to return the full path in CROSS_COMPILE 936 rather than adding it to the PATH variable 937 in_env (dict): Input environment to use, None to use current env 938 939 Returns: 940 tuple: 941 dict: Changes that MakeEnvironment has made to the environment 942 key: Environment variable that was changed 943 value: New value (for PATH this only includes components 944 which were added) 945 str: Full value of the new PATH variable 946 """ 947 env = tchn.MakeEnvironment(full_path, env=in_env) 948 949 # Get the original environment 950 orig_env = dict(os.environb if in_env is None else in_env) 951 orig_path = orig_env[b'PATH'].split(b':') 952 953 # Find new variables 954 diff = dict((k, env[k]) for k in env if orig_env.get(k) != env[k]) 955 956 # Find new / different path components 957 diff_path = None 958 new_path = None 959 if b'PATH' in diff: 960 new_path = diff[b'PATH'].split(b':') 961 diff_paths = [p for p in new_path if p not in orig_path] 962 diff_path = b':'.join(p for p in new_path if p not in orig_path) 963 if diff_path: 964 diff[b'PATH'] = diff_path 965 else: 966 del diff[b'PATH'] 967 return diff, new_path 968 969 def test_toolchain_env(self): 970 """Test PATH and other environment settings for toolchains""" 971 # Use a toolchain which has a path, so that full_path makes a difference 972 tchn = self.toolchains.Select('aarch64') 973 974 # Normal cases 975 diff = self.call_make_environment(tchn, full_path=False)[0] 976 self.assertEqual( 977 {b'CROSS_COMPILE': b'aarch64-linux-', b'LC_ALL': b'C', 978 b'PATH': b'/path/to'}, diff) 979 980 diff = self.call_make_environment(tchn, full_path=True)[0] 981 self.assertEqual( 982 {b'CROSS_COMPILE': b'/path/to/aarch64-linux-', b'LC_ALL': b'C'}, 983 diff) 984 985 # When overriding the toolchain, only LC_ALL should be set 986 tchn.override_toolchain = True 987 diff = self.call_make_environment(tchn, full_path=True)[0] 988 self.assertEqual({b'LC_ALL': b'C'}, diff) 989 990 # Test that Python sandbox is handled correctly 991 tchn.override_toolchain = False 992 sys.prefix = '/some/venv' 993 env = dict(os.environb) 994 env[b'PATH'] = b'/some/venv/bin:other/things' 995 tchn.path = '/my/path' 996 diff, diff_path = self.call_make_environment(tchn, False, env) 997 998 self.assertIn(b'PATH', diff) 999 self.assertEqual([b'/some/venv/bin', b'/my/path', b'other/things'], 1000 diff_path) 1001 self.assertEqual( 1002 {b'CROSS_COMPILE': b'aarch64-linux-', b'LC_ALL': b'C', 1003 b'PATH': b'/my/path'}, diff) 1004 1005 # Handle a toolchain wrapper 1006 tchn.path = '' 1007 bsettings.add_section('toolchain-wrapper') 1008 bsettings.set_item('toolchain-wrapper', 'my-wrapper', 'fred') 1009 diff = self.call_make_environment(tchn, full_path=True)[0] 1010 self.assertEqual( 1011 {b'CROSS_COMPILE': b'fred aarch64-linux-', b'LC_ALL': b'C'}, diff) 1012 1013 def test_skip_dtc(self): 1014 """Test skipping building the dtc tool""" 1015 old_path = os.getenv('PATH') 1016 try: 1017 os.environ['PATH'] = self.base_dir 1018 1019 # Check a missing tool 1020 with self.assertRaises(ValueError) as exc: 1021 builder.Builder(self.toolchains, self.base_dir, None, 0, 2, 1022 dtc_skip=True) 1023 self.assertIn('Cannot find dtc', str(exc.exception)) 1024 1025 # Create a fake tool to use 1026 dtc = os.path.join(self.base_dir, 'dtc') 1027 tools.write_file(dtc, b'xx') 1028 os.chmod(dtc, 0o777) 1029 1030 build = builder.Builder(self.toolchains, self.base_dir, None, 0, 2, 1031 dtc_skip=True) 1032 toolchain = self.toolchains.Select('arm') 1033 env = build.make_environment(toolchain) 1034 self.assertIn(b'DTC', env) 1035 1036 # Try the normal case, i.e. not skipping the dtc build 1037 build = builder.Builder(self.toolchains, self.base_dir, None, 0, 2) 1038 toolchain = self.toolchains.Select('arm') 1039 env = build.make_environment(toolchain) 1040 self.assertNotIn(b'DTC', env) 1041 finally: 1042 os.environ['PATH'] = old_path 1043 1044 def testHomedir(self): 1045 """Test using ~ in a toolchain or toolchain-prefix section""" 1046 # Add some test settings 1047 bsettings.setup(None) 1048 bsettings.add_file(settings_data_homedir) 1049 1050 # Set up the toolchains 1051 home = os.path.expanduser('~') 1052 toolchains = toolchain.Toolchains() 1053 toolchains.GetSettings() 1054 self.assertEqual([f'{home}/mypath'], toolchains.paths) 1055 1056 # Check scanning 1057 with terminal.capture() as (stdout, _): 1058 toolchains.Scan(verbose=True, raise_on_error=False) 1059 lines = iter(stdout.getvalue().splitlines() + ['##done']) 1060 self.assertEqual('Scanning for tool chains', next(lines)) 1061 self.assertEqual(f" - scanning prefix '{home}/mypath-x86-'", 1062 next(lines)) 1063 self.assertEqual( 1064 f"Error: No tool chain found for prefix '{home}/mypath-x86-gcc'", 1065 next(lines)) 1066 self.assertEqual(f" - scanning path '{home}/mypath'", next(lines)) 1067 self.assertEqual(f" - looking in '{home}/mypath/.'", next(lines)) 1068 self.assertEqual(f" - looking in '{home}/mypath/bin'", next(lines)) 1069 self.assertEqual(f" - looking in '{home}/mypath/usr/bin'", 1070 next(lines)) 1071 self.assertEqual('##done', next(lines)) 1072 1073 # Check adding a toolchain 1074 with terminal.capture() as (stdout, _): 1075 toolchains.Add('~/aarch64-linux-gcc', test=True, verbose=True) 1076 lines = iter(stdout.getvalue().splitlines() + ['##done']) 1077 self.assertEqual('Tool chain test: BAD', next(lines)) 1078 self.assertEqual(f'Command: {home}/aarch64-linux-gcc --version', 1079 next(lines)) 1080 self.assertEqual('', next(lines)) 1081 self.assertEqual('', next(lines)) 1082 self.assertEqual('##done', next(lines)) 1083 1084 1085if __name__ == "__main__": 1086 unittest.main() 1087