1#!/usr/bin/env python3 2# Copyright (c) 2024 Intel Corporation 3# 4# SPDX-License-Identifier: Apache-2.0 5""" 6Blackbox tests for twister's command line functions related to memory footprints. 7""" 8 9import importlib 10import json 11from unittest import mock 12import os 13import pytest 14import sys 15import re 16 17# pylint: disable=no-name-in-module 18from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test 19from twisterlib.statuses import TwisterStatus 20from twisterlib.testplan import TestPlan 21 22 23@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) 24class TestFootprint: 25 # Log printed when entering delta calculations 26 FOOTPRINT_LOG = 'running footprint_reports' 27 28 # These warnings notify us that deltas were shown in log. 29 # Coupled with the code under test. 30 DELTA_WARNING_COMPARE = re.compile( 31 r'Found [1-9]+[0-9]* footprint deltas to .*blackbox-out\.[0-9]+/twister.json as a baseline' 32 ) 33 DELTA_WARNING_RUN = re.compile(r'Found [1-9]+[0-9]* footprint deltas to the last twister run') 34 35 # Size report key we modify to control for deltas 36 RAM_KEY = 'used_ram' 37 DELTA_DETAIL = re.compile(RAM_KEY + r' \+[0-9]+, is now +[0-9]+ \+[0-9.]+%') 38 39 @classmethod 40 def setup_class(cls): 41 apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister') 42 cls.loader = importlib.machinery.SourceFileLoader('__main__', apath) 43 cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader) 44 cls.twister_module = importlib.util.module_from_spec(cls.spec) 45 46 @classmethod 47 def teardown_class(cls): 48 pass 49 50 @pytest.mark.parametrize( 51 'old_ram_multiplier, expect_delta_log', 52 [ 53 (0.75, True), 54 (1.25, False), 55 ], 56 ids=['footprint increased', 'footprint reduced'] 57 ) 58 def test_compare_report(self, caplog, out_path, old_ram_multiplier, expect_delta_log): 59 # First run 60 test_platforms = ['intel_adl_crb'] 61 path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group') 62 args = ['-i', '--outdir', out_path, '-T', path] + \ 63 ['--enable-size-report'] + \ 64 [val for pair in zip( 65 ['-p'] * len(test_platforms), test_platforms 66 ) for val in pair] 67 68 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 69 pytest.raises(SystemExit) as sys_exit: 70 self.loader.exec_module(self.twister_module) 71 72 assert str(sys_exit.value) == '0' 73 74 # Modify the older report so we can control the difference. 75 # Note: if footprint tests take too long, replace first run with a prepared twister.json 76 # That will increase test-to-code_under_test coupling, however. 77 with open(os.path.join(out_path, 'twister.json')) as f: 78 j = json.load(f) 79 for ts in j['testsuites']: 80 if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN: 81 # We assume positive RAM usage. 82 ts[self.RAM_KEY] *= old_ram_multiplier 83 84 with open(os.path.join(out_path, 'twister.json'), 'w') as f: 85 f.write(json.dumps(j, indent=4)) 86 87 report_path = os.path.join( 88 os.path.dirname(out_path), 89 f'{os.path.basename(out_path)}.1', 90 'twister.json' 91 ) 92 93 # Second run 94 test_platforms = ['intel_adl_crb'] 95 path = os.path.join(TEST_DATA, 'tests', 'dummy') 96 args = ['-i', '--outdir', out_path, '-T', path] + \ 97 ['--compare-report', report_path] + \ 98 ['--show-footprint'] + \ 99 [val for pair in zip( 100 ['-p'] * len(test_platforms), test_platforms 101 ) for val in pair] 102 103 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 104 pytest.raises(SystemExit) as sys_exit: 105 self.loader.exec_module(self.twister_module) 106 107 assert str(sys_exit.value) == '0' 108 109 assert self.FOOTPRINT_LOG in caplog.text 110 111 if expect_delta_log: 112 assert self.RAM_KEY in caplog.text 113 assert re.search(self.DELTA_WARNING_COMPARE, caplog.text), \ 114 'Expected footprint deltas not logged.' 115 else: 116 assert self.RAM_KEY not in caplog.text 117 assert not re.search(self.DELTA_WARNING_COMPARE, caplog.text), \ 118 'Unexpected footprint deltas logged.' 119 120 def test_footprint_from_buildlog(self, out_path): 121 # First run 122 test_platforms = ['intel_adl_crb'] 123 path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group') 124 args = ['-i', '--outdir', out_path, '-T', path] + \ 125 [] + \ 126 ['--enable-size-report'] + \ 127 [val for pair in zip( 128 ['-p'] * len(test_platforms), test_platforms 129 ) for val in pair] 130 131 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 132 pytest.raises(SystemExit) as sys_exit: 133 self.loader.exec_module(self.twister_module) 134 135 assert str(sys_exit.value) == '0' 136 137 # Get values 138 old_values = [] 139 with open(os.path.join(out_path, 'twister.json')) as f: 140 j = json.load(f) 141 for ts in j['testsuites']: 142 if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN: 143 assert self.RAM_KEY in ts 144 old_values += [ts[self.RAM_KEY]] 145 146 # Second run 147 test_platforms = ['intel_adl_crb'] 148 path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group') 149 args = ['-i', '--outdir', out_path, '-T', path] + \ 150 ['--footprint-from-buildlog'] + \ 151 ['--enable-size-report'] + \ 152 [val for pair in zip( 153 ['-p'] * len(test_platforms), test_platforms 154 ) for val in pair] 155 156 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 157 pytest.raises(SystemExit) as sys_exit: 158 self.loader.exec_module(self.twister_module) 159 160 assert str(sys_exit.value) == '0' 161 162 # Get values 163 new_values = [] 164 with open(os.path.join(out_path, 'twister.json')) as f: 165 j = json.load(f) 166 for ts in j['testsuites']: 167 if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN: 168 assert self.RAM_KEY in ts 169 new_values += [ts[self.RAM_KEY]] 170 171 # There can be false positives if our calculations become too accurate. 172 # Turn this test into a dummy (check only exit value) in such case. 173 assert sorted(old_values) != sorted(new_values), \ 174 'Same values whether calculating size or using buildlog.' 175 176 @pytest.mark.parametrize( 177 'old_ram_multiplier, threshold, expect_delta_log', 178 [ 179 (0.75, 95, False), 180 (0.75, 25, True), 181 ], 182 ids=['footprint threshold not met', 'footprint threshold met'] 183 ) 184 def test_footprint_threshold(self, caplog, out_path, old_ram_multiplier, 185 threshold, expect_delta_log): 186 # First run 187 test_platforms = ['intel_adl_crb'] 188 path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group') 189 args = ['-i', '--outdir', out_path, '-T', path] + \ 190 ['--enable-size-report'] + \ 191 [val for pair in zip( 192 ['-p'] * len(test_platforms), test_platforms 193 ) for val in pair] 194 195 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 196 pytest.raises(SystemExit) as sys_exit: 197 self.loader.exec_module(self.twister_module) 198 199 assert str(sys_exit.value) == '0' 200 201 # Modify the older report so we can control the difference. 202 # Note: if footprint tests take too long, replace first run with a prepared twister.json 203 # That will increase test-to-code_under_test coupling, however. 204 with open(os.path.join(out_path, 'twister.json')) as f: 205 j = json.load(f) 206 for ts in j['testsuites']: 207 if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN: 208 # We assume positive RAM usage. 209 ts[self.RAM_KEY] *= old_ram_multiplier 210 with open(os.path.join(out_path, 'twister.json'), 'w') as f: 211 f.write(json.dumps(j, indent=4)) 212 213 report_path = os.path.join( 214 os.path.dirname(out_path), 215 f'{os.path.basename(out_path)}.1', 216 'twister.json' 217 ) 218 219 # Second run 220 test_platforms = ['intel_adl_crb'] 221 path = os.path.join(TEST_DATA, 'tests', 'dummy') 222 args = ['-i', '--outdir', out_path, '-T', path] + \ 223 [f'--footprint-threshold={threshold}'] + \ 224 ['--compare-report', report_path, '--show-footprint'] + \ 225 [val for pair in zip( 226 ['-p'] * len(test_platforms), test_platforms 227 ) for val in pair] 228 229 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 230 pytest.raises(SystemExit) as sys_exit: 231 self.loader.exec_module(self.twister_module) 232 233 assert str(sys_exit.value) == '0' 234 235 assert self.FOOTPRINT_LOG in caplog.text 236 237 if expect_delta_log: 238 assert self.RAM_KEY in caplog.text 239 assert re.search(self.DELTA_WARNING_COMPARE, caplog.text), \ 240 'Expected footprint deltas not logged.' 241 else: 242 assert self.RAM_KEY not in caplog.text 243 assert not re.search(self.DELTA_WARNING_COMPARE, caplog.text), \ 244 'Unexpected footprint deltas logged.' 245 246 @pytest.mark.parametrize( 247 'flags, old_ram_multiplier, expect_delta_log', 248 [ 249 ([], 0.75, False), 250 (['--show-footprint'], 0.75, True), 251 ], 252 ids=['footprint reduced, no show', 'footprint reduced, show'] 253 ) 254 def test_show_footprint(self, caplog, out_path, flags, old_ram_multiplier, expect_delta_log): 255 # First run 256 test_platforms = ['intel_adl_crb'] 257 path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group') 258 args = ['-i', '--outdir', out_path, '-T', path] + \ 259 ['--enable-size-report'] + \ 260 [val for pair in zip( 261 ['-p'] * len(test_platforms), test_platforms 262 ) for val in pair] 263 264 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 265 pytest.raises(SystemExit) as sys_exit: 266 self.loader.exec_module(self.twister_module) 267 268 assert str(sys_exit.value) == '0' 269 270 # Modify the older report so we can control the difference. 271 # Note: if footprint tests take too long, replace first run with a prepared twister.json 272 # That will increase test-to-code_under_test coupling, however. 273 with open(os.path.join(out_path, 'twister.json')) as f: 274 j = json.load(f) 275 for ts in j['testsuites']: 276 if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN: 277 # We assume positive RAM usage. 278 ts[self.RAM_KEY] *= old_ram_multiplier 279 with open(os.path.join(out_path, 'twister.json'), 'w') as f: 280 f.write(json.dumps(j, indent=4)) 281 282 report_path = os.path.join( 283 os.path.dirname(out_path), 284 f'{os.path.basename(out_path)}.1', 285 'twister.json' 286 ) 287 288 # Second run 289 test_platforms = ['intel_adl_crb'] 290 path = os.path.join(TEST_DATA, 'tests', 'dummy') 291 args = ['-i', '--outdir', out_path, '-T', path] + \ 292 flags + \ 293 ['--compare-report', report_path] + \ 294 [val for pair in zip( 295 ['-p'] * len(test_platforms), test_platforms 296 ) for val in pair] 297 print(args) 298 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 299 pytest.raises(SystemExit) as sys_exit: 300 self.loader.exec_module(self.twister_module) 301 302 assert str(sys_exit.value) == '0' 303 304 assert self.FOOTPRINT_LOG in caplog.text 305 306 if expect_delta_log: 307 assert self.RAM_KEY in caplog.text 308 assert re.search(self.DELTA_DETAIL, caplog.text), \ 309 'Expected footprint delta not logged.' 310 assert re.search(self.DELTA_WARNING_COMPARE, caplog.text), \ 311 'Expected footprint deltas not logged.' 312 else: 313 assert self.RAM_KEY not in caplog.text 314 assert not re.search(self.DELTA_DETAIL, caplog.text), \ 315 'Expected footprint delta not logged.' 316 assert re.search(self.DELTA_WARNING_COMPARE, caplog.text), \ 317 'Expected footprint deltas logged.' 318 319 @pytest.mark.parametrize( 320 'old_ram_multiplier, expect_delta_log', 321 [ 322 (0.75, True), 323 (1.25, False), 324 ], 325 ids=['footprint increased', 'footprint reduced'] 326 ) 327 def test_last_metrics(self, caplog, out_path, old_ram_multiplier, expect_delta_log): 328 # First run 329 test_platforms = ['intel_adl_crb'] 330 path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group') 331 args = ['-i', '--outdir', out_path, '-T', path] + \ 332 ['--enable-size-report'] + \ 333 [val for pair in zip( 334 ['-p'] * len(test_platforms), test_platforms 335 ) for val in pair] 336 337 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 338 pytest.raises(SystemExit) as sys_exit: 339 self.loader.exec_module(self.twister_module) 340 341 assert str(sys_exit.value) == '0' 342 343 # Modify the older report so we can control the difference. 344 # Note: if footprint tests take too long, replace first run with a prepared twister.json 345 # That will increase test-to-code_under_test coupling, however. 346 with open(os.path.join(out_path, 'twister.json')) as f: 347 j = json.load(f) 348 for ts in j['testsuites']: 349 if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN: 350 # We assume positive RAM usage. 351 ts[self.RAM_KEY] *= old_ram_multiplier 352 with open(os.path.join(out_path, 'twister.json'), 'w') as f: 353 f.write(json.dumps(j, indent=4)) 354 355 report_path = os.path.join( 356 os.path.dirname(out_path), 357 f'{os.path.basename(out_path)}.1', 358 'twister.json' 359 ) 360 361 # Second run 362 test_platforms = ['intel_adl_crb'] 363 path = os.path.join(TEST_DATA, 'tests', 'dummy') 364 args = ['-i', '--outdir', out_path, '-T', path] + \ 365 ['--last-metrics'] + \ 366 ['--show-footprint'] + \ 367 [val for pair in zip( 368 ['-p'] * len(test_platforms), test_platforms 369 ) for val in pair] 370 371 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 372 pytest.raises(SystemExit) as sys_exit: 373 self.loader.exec_module(self.twister_module) 374 375 assert str(sys_exit.value) == '0' 376 377 assert self.FOOTPRINT_LOG in caplog.text 378 379 if expect_delta_log: 380 assert self.RAM_KEY in caplog.text 381 assert re.search(self.DELTA_WARNING_RUN, caplog.text), \ 382 'Expected footprint deltas not logged.' 383 else: 384 assert self.RAM_KEY not in caplog.text 385 assert not re.search(self.DELTA_WARNING_RUN, caplog.text), \ 386 'Unexpected footprint deltas logged.' 387 388 second_logs = caplog.records 389 caplog.clear() 390 clear_log_in_test() 391 392 # Third run 393 test_platforms = ['intel_adl_crb'] 394 path = os.path.join(TEST_DATA, 'tests', 'dummy') 395 args = ['-i', '--outdir', out_path, '-T', path] + \ 396 ['--compare-report', report_path] + \ 397 ['--show-footprint'] + \ 398 [val for pair in zip( 399 ['-p'] * len(test_platforms), test_platforms 400 ) for val in pair] 401 402 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 403 pytest.raises(SystemExit) as sys_exit: 404 self.loader.exec_module(self.twister_module) 405 406 assert str(sys_exit.value) == '0' 407 408 # Since second run should use the same source as the third, we should compare them. 409 delta_logs = [ 410 record.getMessage() for record in second_logs \ 411 if self.RAM_KEY in record.getMessage() 412 ] 413 assert all([log in caplog.text for log in delta_logs]) 414 415 @pytest.mark.parametrize( 416 'old_ram_multiplier, expect_delta_log', 417 [ 418 (0.75, True), 419 (1.00, False), 420 (1.25, True), 421 ], 422 ids=['footprint increased', 'footprint constant', 'footprint reduced'] 423 ) 424 def test_all_deltas(self, caplog, out_path, old_ram_multiplier, expect_delta_log): 425 # First run 426 test_platforms = ['intel_adl_crb'] 427 path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group') 428 args = ['-i', '--outdir', out_path, '-T', path] + \ 429 ['--enable-size-report'] + \ 430 [val for pair in zip( 431 ['-p'] * len(test_platforms), test_platforms 432 ) for val in pair] 433 434 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 435 pytest.raises(SystemExit) as sys_exit: 436 self.loader.exec_module(self.twister_module) 437 438 assert str(sys_exit.value) == '0' 439 440 # Modify the older report so we can control the difference. 441 # Note: if footprint tests take too long, replace first run with a prepared twister.json 442 # That will increase test-to-code_under_test coupling, however. 443 with open(os.path.join(out_path, 'twister.json')) as f: 444 j = json.load(f) 445 for ts in j['testsuites']: 446 if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN: 447 # We assume positive RAM usage. 448 ts[self.RAM_KEY] *= old_ram_multiplier 449 with open(os.path.join(out_path, 'twister.json'), 'w') as f: 450 f.write(json.dumps(j, indent=4)) 451 452 report_path = os.path.join( 453 os.path.dirname(out_path), 454 f'{os.path.basename(out_path)}.1', 455 'twister.json' 456 ) 457 458 # Second run 459 test_platforms = ['intel_adl_crb'] 460 path = os.path.join(TEST_DATA, 'tests', 'dummy') 461 args = ['-i', '--outdir', out_path, '-T', path] + \ 462 ['--all-deltas'] + \ 463 ['--compare-report', report_path, '--show-footprint'] + \ 464 [val for pair in zip( 465 ['-p'] * len(test_platforms), test_platforms 466 ) for val in pair] 467 468 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ 469 pytest.raises(SystemExit) as sys_exit: 470 self.loader.exec_module(self.twister_module) 471 472 assert str(sys_exit.value) == '0' 473 474 assert self.FOOTPRINT_LOG in caplog.text 475 476 if expect_delta_log: 477 assert self.RAM_KEY in caplog.text 478 assert re.search(self.DELTA_WARNING_COMPARE, caplog.text), \ 479 'Expected footprint deltas not logged.' 480 else: 481 assert self.RAM_KEY not in caplog.text 482 assert not re.search(self.DELTA_WARNING_COMPARE, caplog.text), \ 483 'Unexpected footprint deltas logged.' 484