1#!/usr/bin/env python3 2# 3# Copyright (c) 2019, Nordic Semiconductor ASA 4# 5# SPDX-License-Identifier: Apache-2.0 6 7'''Tool for parsing a list of projects to determine if they are Zephyr 8projects. If no projects are given then the output from `west list` will be 9used as project list. 10 11Include file is generated for Kconfig using --kconfig-out. 12A <name>:<path> text file is generated for use with CMake using --cmake-out. 13 14Using --twister-out <filename> an argument file for twister script will 15be generated which would point to test and sample roots available in modules 16that can be included during a twister run. This allows testing code 17maintained in modules in addition to what is available in the main Zephyr tree. 18''' 19 20import argparse 21import hashlib 22import os 23import re 24import subprocess 25import sys 26import yaml 27import pykwalify.core 28from pathlib import Path, PurePath, PurePosixPath 29from collections import namedtuple 30 31try: 32 from yaml import CSafeLoader as SafeLoader 33except ImportError: 34 from yaml import SafeLoader 35 36METADATA_SCHEMA = ''' 37## A pykwalify schema for basic validation of the structure of a 38## metadata YAML file. 39## 40# The zephyr/module.yml file is a simple list of key value pairs to be used by 41# the build system. 42type: map 43mapping: 44 name: 45 required: false 46 type: str 47 build: 48 required: false 49 type: map 50 mapping: 51 cmake: 52 required: false 53 type: str 54 kconfig: 55 required: false 56 type: str 57 cmake-ext: 58 required: false 59 type: bool 60 default: false 61 kconfig-ext: 62 required: false 63 type: bool 64 default: false 65 sysbuild-cmake: 66 required: false 67 type: str 68 sysbuild-kconfig: 69 required: false 70 type: str 71 sysbuild-cmake-ext: 72 required: false 73 type: bool 74 default: false 75 sysbuild-kconfig-ext: 76 required: false 77 type: bool 78 default: false 79 depends: 80 required: false 81 type: seq 82 sequence: 83 - type: str 84 settings: 85 required: false 86 type: map 87 mapping: 88 board_root: 89 required: false 90 type: str 91 dts_root: 92 required: false 93 type: str 94 snippet_root: 95 required: false 96 type: str 97 soc_root: 98 required: false 99 type: str 100 arch_root: 101 required: false 102 type: str 103 module_ext_root: 104 required: false 105 type: str 106 sca_root: 107 required: false 108 type: str 109 tests: 110 required: false 111 type: seq 112 sequence: 113 - type: str 114 samples: 115 required: false 116 type: seq 117 sequence: 118 - type: str 119 boards: 120 required: false 121 type: seq 122 sequence: 123 - type: str 124 blobs: 125 required: false 126 type: seq 127 sequence: 128 - type: map 129 mapping: 130 path: 131 required: true 132 type: str 133 sha256: 134 required: true 135 type: str 136 type: 137 required: true 138 type: str 139 enum: ['img', 'lib'] 140 version: 141 required: true 142 type: str 143 license-path: 144 required: true 145 type: str 146 click-through: 147 required: false 148 type: bool 149 default: false 150 url: 151 required: true 152 type: str 153 description: 154 required: true 155 type: str 156 doc-url: 157 required: false 158 type: str 159 security: 160 required: false 161 type: map 162 mapping: 163 external-references: 164 required: false 165 type: seq 166 sequence: 167 - type: str 168 package-managers: 169 required: false 170 type: map 171 mapping: 172 pip: 173 required: false 174 type: map 175 mapping: 176 requirement-files: 177 required: false 178 type: seq 179 sequence: 180 - type: str 181 runners: 182 required: false 183 type: seq 184 sequence: 185 - type: map 186 mapping: 187 file: 188 required: true 189 type: str 190''' 191 192MODULE_YML_PATH = PurePath('zephyr/module.yml') 193# Path to the blobs folder 194MODULE_BLOBS_PATH = PurePath('zephyr/blobs') 195BLOB_PRESENT = 'A' 196BLOB_NOT_PRESENT = 'D' 197BLOB_OUTDATED = 'M' 198 199schema = yaml.load(METADATA_SCHEMA, Loader=SafeLoader) 200 201 202def validate_setting(setting, module_path, filename=None): 203 if setting is not None: 204 if filename is not None: 205 checkfile = Path(module_path) / setting / filename 206 else: 207 checkfile = Path(module_path) / setting 208 if not checkfile.resolve().is_file(): 209 return False 210 return True 211 212 213def process_module(module): 214 module_path = PurePath(module) 215 216 # The input is a module if zephyr/module.{yml,yaml} is a valid yaml file 217 # or if both zephyr/CMakeLists.txt and zephyr/Kconfig are present. 218 219 for module_yml in [module_path / MODULE_YML_PATH, 220 module_path / MODULE_YML_PATH.with_suffix('.yaml')]: 221 if Path(module_yml).is_file(): 222 with Path(module_yml).open('r', encoding='utf-8') as f: 223 meta = yaml.load(f.read(), Loader=SafeLoader) 224 225 try: 226 pykwalify.core.Core(source_data=meta, schema_data=schema)\ 227 .validate() 228 except pykwalify.errors.SchemaError as e: 229 sys.exit('ERROR: Malformed "build" section in file: {}\n{}' 230 .format(module_yml.as_posix(), e)) 231 232 meta['name'] = meta.get('name', module_path.name) 233 meta['name-sanitized'] = re.sub('[^a-zA-Z0-9]', '_', meta['name']) 234 return meta 235 236 if Path(module_path.joinpath('zephyr/CMakeLists.txt')).is_file() and \ 237 Path(module_path.joinpath('zephyr/Kconfig')).is_file(): 238 return {'name': module_path.name, 239 'name-sanitized': re.sub('[^a-zA-Z0-9]', '_', module_path.name), 240 'build': {'cmake': 'zephyr', 'kconfig': 'zephyr/Kconfig'}} 241 242 return None 243 244 245def process_cmake(module, meta): 246 section = meta.get('build', dict()) 247 module_path = PurePath(module) 248 module_yml = module_path.joinpath('zephyr/module.yml') 249 250 cmake_extern = section.get('cmake-ext', False) 251 if cmake_extern: 252 return('\"{}\":\"{}\":\"{}\"\n' 253 .format(meta['name'], 254 module_path.as_posix(), 255 "${ZEPHYR_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}")) 256 257 cmake_setting = section.get('cmake', None) 258 if not validate_setting(cmake_setting, module, 'CMakeLists.txt'): 259 sys.exit('ERROR: "cmake" key in {} has folder value "{}" which ' 260 'does not contain a CMakeLists.txt file.' 261 .format(module_yml.as_posix(), cmake_setting)) 262 263 cmake_path = os.path.join(module, cmake_setting or 'zephyr') 264 cmake_file = os.path.join(cmake_path, 'CMakeLists.txt') 265 if os.path.isfile(cmake_file): 266 return('\"{}\":\"{}\":\"{}\"\n' 267 .format(meta['name'], 268 module_path.as_posix(), 269 Path(cmake_path).resolve().as_posix())) 270 else: 271 return('\"{}\":\"{}\":\"\"\n' 272 .format(meta['name'], 273 module_path.as_posix())) 274 275 276def process_sysbuildcmake(module, meta): 277 section = meta.get('build', dict()) 278 module_path = PurePath(module) 279 module_yml = module_path.joinpath('zephyr/module.yml') 280 281 cmake_extern = section.get('sysbuild-cmake-ext', False) 282 if cmake_extern: 283 return('\"{}\":\"{}\":\"{}\"\n' 284 .format(meta['name'], 285 module_path.as_posix(), 286 "${SYSBUILD_" + meta['name-sanitized'].upper() + "_CMAKE_DIR}")) 287 288 cmake_setting = section.get('sysbuild-cmake', None) 289 if not validate_setting(cmake_setting, module, 'CMakeLists.txt'): 290 sys.exit('ERROR: "cmake" key in {} has folder value "{}" which ' 291 'does not contain a CMakeLists.txt file.' 292 .format(module_yml.as_posix(), cmake_setting)) 293 294 if cmake_setting is None: 295 return "" 296 297 cmake_path = os.path.join(module, cmake_setting or 'zephyr') 298 cmake_file = os.path.join(cmake_path, 'CMakeLists.txt') 299 if os.path.isfile(cmake_file): 300 return('\"{}\":\"{}\":\"{}\"\n' 301 .format(meta['name'], 302 module_path.as_posix(), 303 Path(cmake_path).resolve().as_posix())) 304 else: 305 return('\"{}\":\"{}\":\"\"\n' 306 .format(meta['name'], 307 module_path.as_posix())) 308 309 310def process_settings(module, meta): 311 section = meta.get('build', dict()) 312 build_settings = section.get('settings', None) 313 out_text = "" 314 315 if build_settings is not None: 316 for root in ['board', 'dts', 'snippet', 'soc', 'arch', 'module_ext', 'sca']: 317 setting = build_settings.get(root+'_root', None) 318 if setting is not None: 319 root_path = PurePath(module) / setting 320 out_text += f'"{root.upper()}_ROOT":' 321 out_text += f'"{root_path.as_posix()}"\n' 322 323 return out_text 324 325 326def get_blob_status(path, sha256): 327 if not path.is_file(): 328 return BLOB_NOT_PRESENT 329 with path.open('rb') as f: 330 m = hashlib.sha256() 331 m.update(f.read()) 332 if sha256.lower() == m.hexdigest(): 333 return BLOB_PRESENT 334 else: 335 return BLOB_OUTDATED 336 337 338def process_blobs(module, meta): 339 blobs = [] 340 mblobs = meta.get('blobs', None) 341 if not mblobs: 342 return blobs 343 344 blobs_path = Path(module) / MODULE_BLOBS_PATH 345 for blob in mblobs: 346 blob['module'] = meta.get('name', None) 347 blob['abspath'] = blobs_path / Path(blob['path']) 348 blob['license-abspath'] = Path(module) / Path(blob['license-path']) 349 blob['status'] = get_blob_status(blob['abspath'], blob['sha256']) 350 blobs.append(blob) 351 352 return blobs 353 354 355def kconfig_module_opts(name_sanitized, blobs, taint_blobs): 356 snippet = [f'config ZEPHYR_{name_sanitized.upper()}_MODULE', 357 ' bool', 358 ' default y'] 359 360 if taint_blobs: 361 snippet += [' select TAINT_BLOBS'] 362 363 if blobs: 364 snippet += [f'\nconfig ZEPHYR_{name_sanitized.upper()}_MODULE_BLOBS', 365 ' bool'] 366 if taint_blobs: 367 snippet += [' default y'] 368 369 return snippet 370 371 372def kconfig_snippet(meta, path, kconfig_file=None, blobs=False, taint_blobs=False, sysbuild=False): 373 name = meta['name'] 374 name_sanitized = meta['name-sanitized'] 375 376 snippet = [f'menu "{name} ({path.as_posix()})"'] 377 378 snippet += [f'osource "{kconfig_file.resolve().as_posix()}"' if kconfig_file 379 else f'osource "$(SYSBUILD_{name_sanitized.upper()}_KCONFIG)"' if sysbuild is True 380 else f'osource "$(ZEPHYR_{name_sanitized.upper()}_KCONFIG)"'] 381 382 snippet += kconfig_module_opts(name_sanitized, blobs, taint_blobs) 383 384 snippet += ['endmenu\n'] 385 386 return '\n'.join(snippet) 387 388 389def process_kconfig(module, meta): 390 blobs = process_blobs(module, meta) 391 taint_blobs = any(b['status'] != BLOB_NOT_PRESENT for b in blobs) 392 section = meta.get('build', dict()) 393 module_path = PurePath(module) 394 module_yml = module_path.joinpath('zephyr/module.yml') 395 kconfig_extern = section.get('kconfig-ext', False) 396 name_sanitized = meta['name-sanitized'] 397 snippet = f'ZEPHYR_{name_sanitized.upper()}_MODULE_DIR := {module_path.as_posix()}\n' 398 399 if kconfig_extern: 400 return snippet + kconfig_snippet(meta, module_path, blobs=blobs, taint_blobs=taint_blobs) 401 402 kconfig_setting = section.get('kconfig', None) 403 if not validate_setting(kconfig_setting, module): 404 sys.exit('ERROR: "kconfig" key in {} has value "{}" which does ' 405 'not point to a valid Kconfig file.' 406 .format(module_yml, kconfig_setting)) 407 408 kconfig_file = os.path.join(module, kconfig_setting or 'zephyr/Kconfig') 409 if os.path.isfile(kconfig_file): 410 return snippet + kconfig_snippet(meta, module_path, Path(kconfig_file), 411 blobs=blobs, taint_blobs=taint_blobs) 412 else: 413 return snippet + '\n'.join(kconfig_module_opts(name_sanitized, blobs, taint_blobs)) + '\n' 414 415 416def process_sysbuildkconfig(module, meta): 417 section = meta.get('build', dict()) 418 module_path = PurePath(module) 419 module_yml = module_path.joinpath('zephyr/module.yml') 420 kconfig_extern = section.get('sysbuild-kconfig-ext', False) 421 name_sanitized = meta['name-sanitized'] 422 snippet = f'ZEPHYR_{name_sanitized.upper()}_MODULE_DIR := {module_path.as_posix()}\n' 423 424 if kconfig_extern: 425 return snippet + kconfig_snippet(meta, module_path, sysbuild=True) 426 427 kconfig_setting = section.get('sysbuild-kconfig', None) 428 if not validate_setting(kconfig_setting, module): 429 sys.exit('ERROR: "kconfig" key in {} has value "{}" which does ' 430 'not point to a valid Kconfig file.' 431 .format(module_yml, kconfig_setting)) 432 433 if kconfig_setting is not None: 434 kconfig_file = os.path.join(module, kconfig_setting) 435 if os.path.isfile(kconfig_file): 436 return snippet + kconfig_snippet(meta, module_path, Path(kconfig_file)) 437 438 return snippet + \ 439 (f'config ZEPHYR_{name_sanitized.upper()}_MODULE\n' 440 f' bool\n' 441 f' default y\n') 442 443 444def process_twister(module, meta): 445 446 out = "" 447 tests = meta.get('tests', []) 448 samples = meta.get('samples', []) 449 boards = meta.get('boards', []) 450 451 for pth in tests + samples: 452 if pth: 453 dir = os.path.join(module, pth) 454 out += '-T\n{}\n'.format(PurePath(os.path.abspath(dir)) 455 .as_posix()) 456 457 for pth in boards: 458 if pth: 459 dir = os.path.join(module, pth) 460 out += '--board-root\n{}\n'.format(PurePath(os.path.abspath(dir)) 461 .as_posix()) 462 463 return out 464 465def is_valid_git_revision(revision): 466 """ 467 Returns True if the given string is a valid git revision hash (40 hex digits). 468 """ 469 if not isinstance(revision, str): 470 return False 471 return bool(re.fullmatch(r'[0-9a-fA-F]{40}', revision)) 472 473def _create_meta_project(project_path): 474 def git_revision(path): 475 rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'], 476 stdout=subprocess.PIPE, 477 stderr=subprocess.PIPE, 478 cwd=path).wait() 479 if rc == 0: 480 # A git repo. 481 popen = subprocess.Popen(['git', 'rev-parse', 'HEAD'], 482 stdout=subprocess.PIPE, 483 stderr=subprocess.PIPE, 484 cwd=path) 485 stdout, stderr = popen.communicate() 486 stdout = stdout.decode('utf-8') 487 488 if not (popen.returncode or stderr): 489 revision = stdout.rstrip() 490 491 rc = subprocess.Popen(['git', 'diff-index', '--quiet', 'HEAD', 492 '--'], 493 stdout=None, 494 stderr=None, 495 cwd=path).wait() 496 if rc: 497 return revision + '-dirty', True 498 return revision, False 499 return "unknown", False 500 501 def git_remote(path): 502 popen = subprocess.Popen(['git', 'remote'], 503 stdout=subprocess.PIPE, 504 stderr=subprocess.PIPE, 505 cwd=path) 506 stdout, stderr = popen.communicate() 507 stdout = stdout.decode('utf-8') 508 509 remotes_name = [] 510 if not (popen.returncode or stderr): 511 remotes_name = stdout.rstrip().split('\n') 512 513 remote_url = None 514 515 # If more than one remote, do not return any remote 516 if len(remotes_name) == 1: 517 remote = remotes_name[0] 518 popen = subprocess.Popen(['git', 'remote', 'get-url', remote], 519 stdout=subprocess.PIPE, 520 stderr=subprocess.PIPE, 521 cwd=path) 522 stdout, stderr = popen.communicate() 523 stdout = stdout.decode('utf-8') 524 525 if not (popen.returncode or stderr): 526 remote_url = stdout.rstrip() 527 528 return remote_url 529 530 def git_tags(path, revision): 531 if not revision or len(revision) == 0: 532 return None 533 534 popen = subprocess.Popen(['git', '-P', 'tag', '--points-at', revision], 535 stdout=subprocess.PIPE, 536 stderr=subprocess.PIPE, 537 cwd=path) 538 stdout, stderr = popen.communicate() 539 stdout = stdout.decode('utf-8') 540 541 tags = None 542 if not (popen.returncode or stderr): 543 tags = stdout.rstrip().splitlines() 544 545 return tags 546 547 workspace_dirty = False 548 path = PurePath(project_path).as_posix() 549 550 revision, dirty = git_revision(path) 551 workspace_dirty |= dirty 552 remote = git_remote(path) 553 tags = git_tags(path, revision) 554 555 meta_project = {'path': path, 556 'revision': revision} 557 558 if remote: 559 meta_project['remote'] = remote 560 561 if tags: 562 meta_project['tags'] = tags 563 564 return meta_project, workspace_dirty 565 566 567def _get_meta_project(meta_projects_list, project_path): 568 projects = [ prj for prj in meta_projects_list[1:] if prj["path"] == project_path ] 569 570 return projects[0] if len(projects) == 1 else None 571 572 573def process_meta(zephyr_base, west_projs, modules, extra_modules=None, 574 propagate_state=False): 575 # Process zephyr_base, projects, and modules and create a dictionary 576 # with meta information for each input. 577 # 578 # The dictionary will contain meta info in the following lists: 579 # - zephyr: path and revision 580 # - modules: name, path, and revision 581 # - west-projects: path and revision 582 # 583 # returns the dictionary with said lists 584 585 meta = {'zephyr': None, 'modules': None, 'workspace': None} 586 587 zephyr_project, zephyr_dirty = _create_meta_project(zephyr_base) 588 zephyr_off = zephyr_project.get("remote") is None 589 590 workspace_dirty = zephyr_dirty 591 workspace_extra = extra_modules is not None 592 workspace_off = zephyr_off 593 594 if zephyr_off and is_valid_git_revision(zephyr_project['revision']): 595 zephyr_project['revision'] += '-off' 596 597 meta['zephyr'] = zephyr_project 598 meta['workspace'] = {} 599 600 if west_projs is not None: 601 from west.manifest import MANIFEST_REV_BRANCH 602 projects = west_projs['projects'] 603 meta_projects = [] 604 605 manifest_path = projects[0].posixpath 606 607 # Special treatment of manifest project 608 # Git information (remote/revision) are not provided by west for the Manifest (west.yml) 609 # To mitigate this, we check if we don't use the manifest from the zephyr repository or an other project. 610 # If it's from zephyr, reuse zephyr information 611 # If it's from an other project, ignore it, it will be added later 612 # If it's not found, we extract data manually (remote/revision) from the directory 613 614 manifest_project = None 615 manifest_dirty = False 616 manifest_off = False 617 618 if zephyr_base == manifest_path: 619 manifest_project = zephyr_project 620 manifest_dirty = zephyr_dirty 621 manifest_off = zephyr_off 622 elif not [ prj for prj in projects[1:] if prj.posixpath == manifest_path ]: 623 manifest_project, manifest_dirty = _create_meta_project( 624 projects[0].posixpath) 625 manifest_off = manifest_project.get("remote") is None 626 if manifest_off and is_valid_git_revision(manifest_project['revision']): 627 manifest_project["revision"] += "-off" 628 629 if manifest_project: 630 workspace_off |= manifest_off 631 workspace_dirty |= manifest_dirty 632 meta_projects.append(manifest_project) 633 634 # Iterates on all projects except the first one (manifest) 635 for project in projects[1:]: 636 meta_project, dirty = _create_meta_project(project.posixpath) 637 workspace_dirty |= dirty 638 meta_projects.append(meta_project) 639 640 off = False 641 if not meta_project.get("remote") or project.sha(MANIFEST_REV_BRANCH) != meta_project['revision'].removesuffix("-dirty"): 642 off = True 643 if not meta_project.get('remote') or project.url != meta_project['remote']: 644 # Force manifest URL and set commit as 'off' 645 meta_project['url'] = project.url 646 off = True 647 648 if off: 649 if is_valid_git_revision(meta_project['revision']): 650 meta_project['revision'] += '-off' 651 workspace_off |= off 652 653 # If manifest is in project, updates related variables 654 if project.posixpath == manifest_path: 655 manifest_dirty |= dirty 656 manifest_off |= off 657 manifest_project = meta_project 658 659 meta.update({'west': {'manifest': west_projs['manifest_path'], 660 'projects': meta_projects}}) 661 meta['workspace'].update({'off': workspace_off}) 662 663 # Iterates on all modules 664 meta_modules = [] 665 for module in modules: 666 # Check if modules is not in projects 667 # It allows to have the "-off" flag since `modules` variable` does not provide URL/remote 668 meta_module = _get_meta_project(meta_projects, module.project) 669 670 if not meta_module: 671 meta_module, dirty = _create_meta_project(module.project) 672 workspace_dirty |= dirty 673 674 meta_module['name'] = module.meta.get('name') 675 676 if module.meta.get('security'): 677 meta_module['security'] = module.meta.get('security') 678 meta_modules.append(meta_module) 679 680 meta['modules'] = meta_modules 681 682 meta['workspace'].update({'dirty': workspace_dirty, 683 'extra': workspace_extra}) 684 685 if propagate_state: 686 zephyr_revision = zephyr_project['revision'] 687 if is_valid_git_revision(zephyr_revision): 688 if workspace_dirty and not zephyr_dirty: 689 zephyr_revision += '-dirty' 690 if workspace_extra: 691 zephyr_revision += '-extra' 692 if workspace_off and not zephyr_off: 693 zephyr_revision += '-off' 694 zephyr_project.update({'revision': zephyr_revision}) 695 696 if west_projs is not None: 697 manifest_revision = manifest_project['revision'] 698 if is_valid_git_revision(manifest_revision): 699 if workspace_dirty and not manifest_dirty: 700 manifest_revision += '-dirty' 701 if workspace_extra: 702 manifest_revision += '-extra' 703 if workspace_off and not manifest_off: 704 manifest_revision += '-off' 705 manifest_project.update({'revision': manifest_revision}) 706 707 return meta 708 709 710def west_projects(manifest=None): 711 manifest_path = None 712 projects = [] 713 # West is imported here, as it is optional 714 # (and thus maybe not installed) 715 # if user is providing a specific modules list. 716 try: 717 from west.manifest import Manifest 718 except ImportError: 719 # West is not installed, so don't return any projects. 720 return None 721 722 # If west *is* installed, we need all of the following imports to 723 # work. West versions that are excessively old may fail here: 724 # west.configuration.MalformedConfig was 725 # west.manifest.MalformedConfig until west v0.14.0, for example. 726 # These should be hard errors. 727 from west.manifest import \ 728 ManifestImportFailed, MalformedManifest, ManifestVersionError 729 from west.configuration import MalformedConfig 730 from west.util import WestNotFound 731 from west.version import __version__ as WestVersion 732 733 from packaging import version 734 try: 735 if not manifest: 736 manifest = Manifest.from_file() 737 if version.parse(WestVersion) >= version.parse('0.9.0'): 738 projects = [p for p in manifest.get_projects([]) 739 if manifest.is_active(p)] 740 else: 741 projects = manifest.get_projects([]) 742 manifest_path = manifest.path 743 return {'manifest_path': manifest_path, 'projects': projects} 744 except (ManifestImportFailed, MalformedManifest, 745 ManifestVersionError, MalformedConfig) as e: 746 sys.exit(f'ERROR: {e}') 747 except WestNotFound: 748 # Only accept WestNotFound, meaning we are not in a west 749 # workspace. Such setup is allowed, as west may be installed 750 # but the project is not required to use west. 751 pass 752 return None 753 754 755def parse_modules(zephyr_base, manifest=None, west_projs=None, modules=None, 756 extra_modules=None): 757 758 if modules is None: 759 west_projs = west_projs or west_projects(manifest) 760 modules = ([p.posixpath for p in west_projs['projects']] 761 if west_projs else []) 762 763 if extra_modules is None: 764 extra_modules = [] 765 for var in ['EXTRA_ZEPHYR_MODULES', 'ZEPHYR_EXTRA_MODULES']: 766 extra_module = os.environ.get(var, None) 767 if not extra_module: 768 continue 769 extra_modules.extend(PurePosixPath(p) for p in extra_module.split(';')) 770 771 Module = namedtuple('Module', ['project', 'meta', 'depends']) 772 773 all_modules_by_name = {} 774 # dep_modules is a list of all modules that has an unresolved dependency 775 dep_modules = [] 776 # start_modules is a list modules with no depends left (no incoming edge) 777 start_modules = [] 778 # sorted_modules is a topological sorted list of the modules 779 sorted_modules = [] 780 781 for project in modules + extra_modules: 782 # Avoid including Zephyr base project as module. 783 if project == zephyr_base: 784 continue 785 786 meta = process_module(project) 787 if meta: 788 depends = meta.get('build', {}).get('depends', []) 789 all_modules_by_name[meta['name']] = Module(project, meta, depends) 790 791 elif project in extra_modules: 792 sys.exit(f'{project}, given in ZEPHYR_EXTRA_MODULES, ' 793 'is not a valid zephyr module') 794 795 for module in all_modules_by_name.values(): 796 if not module.depends: 797 start_modules.append(module) 798 else: 799 dep_modules.append(module) 800 801 # This will do a topological sort to ensure the modules are ordered 802 # according to dependency settings. 803 while start_modules: 804 node = start_modules.pop(0) 805 sorted_modules.append(node) 806 node_name = node.meta['name'] 807 to_remove = [] 808 for module in dep_modules: 809 if node_name in module.depends: 810 module.depends.remove(node_name) 811 if not module.depends: 812 start_modules.append(module) 813 to_remove.append(module) 814 for module in to_remove: 815 dep_modules.remove(module) 816 817 if dep_modules: 818 # If there are any modules with unresolved dependencies, then the 819 # modules contains unmet or cyclic dependencies. Error out. 820 error = 'Unmet or cyclic dependencies in modules:\n' 821 for module in dep_modules: 822 error += f'{module.project} depends on: {module.depends}\n' 823 sys.exit(error) 824 825 return sorted_modules 826 827 828def main(): 829 parser = argparse.ArgumentParser(description=''' 830 Process a list of projects and create Kconfig / CMake include files for 831 projects which are also a Zephyr module''', allow_abbrev=False) 832 833 parser.add_argument('--kconfig-out', 834 help="""File to write with resulting KConfig import 835 statements.""") 836 parser.add_argument('--twister-out', 837 help="""File to write with resulting twister 838 parameters.""") 839 parser.add_argument('--cmake-out', 840 help="""File to write with resulting <name>:<path> 841 values to use for including in CMake""") 842 parser.add_argument('--sysbuild-kconfig-out', 843 help="""File to write with resulting KConfig import 844 statements.""") 845 parser.add_argument('--sysbuild-cmake-out', 846 help="""File to write with resulting <name>:<path> 847 values to use for including in CMake""") 848 parser.add_argument('--meta-out', 849 help="""Write a build meta YaML file containing a list 850 of Zephyr modules and west projects. 851 If a module or project is also a git repository 852 the current SHA revision will also be written.""") 853 parser.add_argument('--meta-state-propagate', action='store_true', 854 help="""Propagate state of modules and west projects 855 to the suffix of the Zephyr SHA and if west is 856 used, to the suffix of the manifest SHA""") 857 parser.add_argument('--settings-out', 858 help="""File to write with resulting <name>:<value> 859 values to use for including in CMake""") 860 parser.add_argument('-m', '--modules', nargs='+', 861 help="""List of modules to parse instead of using `west 862 list`""") 863 parser.add_argument('-x', '--extra-modules', nargs='+', 864 help='List of extra modules to parse') 865 parser.add_argument('-z', '--zephyr-base', 866 help='Path to zephyr repository') 867 args = parser.parse_args() 868 869 kconfig = "" 870 cmake = "" 871 sysbuild_kconfig = "" 872 sysbuild_cmake = "" 873 settings = "" 874 twister = "" 875 876 west_projs = west_projects() 877 modules = parse_modules(args.zephyr_base, None, west_projs, 878 args.modules, args.extra_modules) 879 880 for module in modules: 881 kconfig += process_kconfig(module.project, module.meta) 882 cmake += process_cmake(module.project, module.meta) 883 sysbuild_kconfig += process_sysbuildkconfig( 884 module.project, module.meta) 885 sysbuild_cmake += process_sysbuildcmake(module.project, module.meta) 886 settings += process_settings(module.project, module.meta) 887 twister += process_twister(module.project, module.meta) 888 889 if args.kconfig_out: 890 with open(args.kconfig_out, 'w', encoding="utf-8") as fp: 891 fp.write(kconfig) 892 893 if args.cmake_out: 894 with open(args.cmake_out, 'w', encoding="utf-8") as fp: 895 fp.write(cmake) 896 897 if args.sysbuild_kconfig_out: 898 with open(args.sysbuild_kconfig_out, 'w', encoding="utf-8") as fp: 899 fp.write(sysbuild_kconfig) 900 901 if args.sysbuild_cmake_out: 902 with open(args.sysbuild_cmake_out, 'w', encoding="utf-8") as fp: 903 fp.write(sysbuild_cmake) 904 905 if args.settings_out: 906 with open(args.settings_out, 'w', encoding="utf-8") as fp: 907 fp.write('''\ 908# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY! 909# 910# This file contains build system settings derived from your modules. 911# 912# Modules may be set via ZEPHYR_MODULES, ZEPHYR_EXTRA_MODULES, 913# and/or the west manifest file. 914# 915# See the Modules guide for more information. 916''') 917 fp.write(settings) 918 919 if args.twister_out: 920 with open(args.twister_out, 'w', encoding="utf-8") as fp: 921 fp.write(twister) 922 923 if args.meta_out: 924 meta = process_meta(args.zephyr_base, west_projs, modules, 925 args.extra_modules, args.meta_state_propagate) 926 927 with open(args.meta_out, 'w', encoding="utf-8") as fp: 928 # Ignore references and insert data instead 929 yaml.Dumper.ignore_aliases = lambda self, data: True 930 fp.write(yaml.dump(meta)) 931 932 933if __name__ == "__main__": 934 main() 935