1#!/usr/bin/env python3 2# 3# This file is part of the MicroPython project, http://micropython.org/ 4# 5# The MIT License (MIT) 6# 7# Copyright (c) 2019 Damien P. George 8# 9# Permission is hereby granted, free of charge, to any person obtaining a copy 10# of this software and associated documentation files (the "Software"), to deal 11# in the Software without restriction, including without limitation the rights 12# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13# copies of the Software, and to permit persons to whom the Software is 14# furnished to do so, subject to the following conditions: 15# 16# The above copyright notice and this permission notice shall be included in 17# all copies or substantial portions of the Software. 18# 19# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 25# THE SOFTWARE. 26 27from __future__ import print_function 28import sys 29import os 30import subprocess 31 32 33########################################################################### 34# Public functions to be used in the manifest 35 36 37def include(manifest, **kwargs): 38 """Include another manifest. 39 40 The manifest argument can be a string (filename) or an iterable of 41 strings. 42 43 Relative paths are resolved with respect to the current manifest file. 44 45 Optional kwargs can be provided which will be available to the 46 included script via the `options` variable. 47 48 e.g. include("path.py", extra_features=True) 49 50 in path.py: 51 options.defaults(standard_features=True) 52 53 # freeze minimal modules. 54 if options.standard_features: 55 # freeze standard modules. 56 if options.extra_features: 57 # freeze extra modules. 58 """ 59 60 if not isinstance(manifest, str): 61 for m in manifest: 62 include(m) 63 else: 64 manifest = convert_path(manifest) 65 with open(manifest) as f: 66 # Make paths relative to this manifest file while processing it. 67 # Applies to includes and input files. 68 prev_cwd = os.getcwd() 69 os.chdir(os.path.dirname(manifest)) 70 exec(f.read(), globals(), {"options": IncludeOptions(**kwargs)}) 71 os.chdir(prev_cwd) 72 73 74def freeze(path, script=None, opt=0): 75 """Freeze the input, automatically determining its type. A .py script 76 will be compiled to a .mpy first then frozen, and a .mpy file will be 77 frozen directly. 78 79 `path` must be a directory, which is the base directory to search for 80 files from. When importing the resulting frozen modules, the name of 81 the module will start after `path`, ie `path` is excluded from the 82 module name. 83 84 If `path` is relative, it is resolved to the current manifest.py. 85 Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need 86 to access specific paths. 87 88 If `script` is None all files in `path` will be frozen. 89 90 If `script` is an iterable then freeze() is called on all items of the 91 iterable (with the same `path` and `opt` passed through). 92 93 If `script` is a string then it specifies the file or directory to 94 freeze, and can include extra directories before the file or last 95 directory. The file or directory will be searched for in `path`. If 96 `script` is a directory then all files in that directory will be frozen. 97 98 `opt` is the optimisation level to pass to mpy-cross when compiling .py 99 to .mpy. 100 """ 101 102 freeze_internal(KIND_AUTO, path, script, opt) 103 104 105def freeze_as_str(path): 106 """Freeze the given `path` and all .py scripts within it as a string, 107 which will be compiled upon import. 108 """ 109 110 freeze_internal(KIND_AS_STR, path, None, 0) 111 112 113def freeze_as_mpy(path, script=None, opt=0): 114 """Freeze the input (see above) by first compiling the .py scripts to 115 .mpy files, then freezing the resulting .mpy files. 116 """ 117 118 freeze_internal(KIND_AS_MPY, path, script, opt) 119 120 121def freeze_mpy(path, script=None, opt=0): 122 """Freeze the input (see above), which must be .mpy files that are 123 frozen directly. 124 """ 125 126 freeze_internal(KIND_MPY, path, script, opt) 127 128 129########################################################################### 130# Internal implementation 131 132KIND_AUTO = 0 133KIND_AS_STR = 1 134KIND_AS_MPY = 2 135KIND_MPY = 3 136 137VARS = {} 138 139manifest_list = [] 140 141 142class IncludeOptions: 143 def __init__(self, **kwargs): 144 self._kwargs = kwargs 145 self._defaults = {} 146 147 def defaults(self, **kwargs): 148 self._defaults = kwargs 149 150 def __getattr__(self, name): 151 return self._kwargs.get(name, self._defaults.get(name, None)) 152 153 154class FreezeError(Exception): 155 pass 156 157 158def system(cmd): 159 try: 160 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 161 return 0, output 162 except subprocess.CalledProcessError as er: 163 return -1, er.output 164 165 166def convert_path(path): 167 # Perform variable substituion. 168 for name, value in VARS.items(): 169 path = path.replace("$({})".format(name), value) 170 # Convert to absolute path (so that future operations don't rely on 171 # still being chdir'ed). 172 return os.path.abspath(path) 173 174 175def get_timestamp(path, default=None): 176 try: 177 stat = os.stat(path) 178 return stat.st_mtime 179 except OSError: 180 if default is None: 181 raise FreezeError("cannot stat {}".format(path)) 182 return default 183 184 185def get_timestamp_newest(path): 186 ts_newest = 0 187 for dirpath, dirnames, filenames in os.walk(path, followlinks=True): 188 for f in filenames: 189 ts_newest = max(ts_newest, get_timestamp(os.path.join(dirpath, f))) 190 return ts_newest 191 192 193def mkdir(filename): 194 path = os.path.dirname(filename) 195 if not os.path.isdir(path): 196 os.makedirs(path) 197 198 199def freeze_internal(kind, path, script, opt): 200 path = convert_path(path) 201 if not os.path.isdir(path): 202 raise FreezeError("freeze path must be a directory: {}".format(path)) 203 if script is None and kind == KIND_AS_STR: 204 if any(f[0] == KIND_AS_STR for f in manifest_list): 205 raise FreezeError("can only freeze one str directory") 206 manifest_list.append((KIND_AS_STR, path, script, opt)) 207 elif script is None or isinstance(script, str) and script.find(".") == -1: 208 # Recursively search `path` for files to freeze, optionally restricted 209 # to a subdirectory specified by `script` 210 if script is None: 211 subdir = "" 212 else: 213 subdir = "/" + script 214 for dirpath, dirnames, filenames in os.walk(path + subdir, followlinks=True): 215 for f in filenames: 216 freeze_internal(kind, path, (dirpath + "/" + f)[len(path) + 1 :], opt) 217 elif not isinstance(script, str): 218 # `script` is an iterable of items to freeze 219 for s in script: 220 freeze_internal(kind, path, s, opt) 221 else: 222 # `script` should specify an individual file to be frozen 223 extension_kind = {KIND_AS_MPY: ".py", KIND_MPY: ".mpy"} 224 if kind == KIND_AUTO: 225 for k, ext in extension_kind.items(): 226 if script.endswith(ext): 227 kind = k 228 break 229 else: 230 print("warn: unsupported file type, skipped freeze: {}".format(script)) 231 return 232 wanted_extension = extension_kind[kind] 233 if not script.endswith(wanted_extension): 234 raise FreezeError("expecting a {} file, got {}".format(wanted_extension, script)) 235 manifest_list.append((kind, path, script, opt)) 236 237 238def main(): 239 # Parse arguments 240 import argparse 241 242 cmd_parser = argparse.ArgumentParser( 243 description="A tool to generate frozen content in MicroPython firmware images." 244 ) 245 cmd_parser.add_argument("-o", "--output", help="output path") 246 cmd_parser.add_argument("-b", "--build-dir", help="output path") 247 cmd_parser.add_argument( 248 "-f", "--mpy-cross-flags", default="", help="flags to pass to mpy-cross" 249 ) 250 cmd_parser.add_argument("-v", "--var", action="append", help="variables to substitute") 251 cmd_parser.add_argument("--mpy-tool-flags", default="", help="flags to pass to mpy-tool") 252 cmd_parser.add_argument("files", nargs="+", help="input manifest list") 253 args = cmd_parser.parse_args() 254 255 # Extract variables for substitution. 256 for var in args.var: 257 name, value = var.split("=", 1) 258 if os.path.exists(value): 259 value = os.path.abspath(value) 260 VARS[name] = value 261 262 if "MPY_DIR" not in VARS or "PORT_DIR" not in VARS: 263 print("MPY_DIR and PORT_DIR variables must be specified") 264 sys.exit(1) 265 266 # Get paths to tools 267 MAKE_FROZEN = VARS["MPY_DIR"] + "/tools/make-frozen.py" 268 MPY_CROSS = VARS["MPY_DIR"] + "/mpy-cross/mpy-cross" 269 if sys.platform == "win32": 270 MPY_CROSS += ".exe" 271 MPY_CROSS = os.getenv("MICROPY_MPYCROSS", MPY_CROSS) 272 MPY_TOOL = VARS["MPY_DIR"] + "/tools/mpy-tool.py" 273 274 # Ensure mpy-cross is built 275 if not os.path.exists(MPY_CROSS): 276 print("mpy-cross not found at {}, please build it first".format(MPY_CROSS)) 277 sys.exit(1) 278 279 # Include top-level inputs, to generate the manifest 280 for input_manifest in args.files: 281 try: 282 if input_manifest.endswith(".py"): 283 include(input_manifest) 284 else: 285 exec(input_manifest) 286 except FreezeError as er: 287 print('freeze error executing "{}": {}'.format(input_manifest, er.args[0])) 288 sys.exit(1) 289 290 # Process the manifest 291 str_paths = [] 292 mpy_files = [] 293 ts_newest = 0 294 for kind, path, script, opt in manifest_list: 295 if kind == KIND_AS_STR: 296 str_paths.append(path) 297 ts_outfile = get_timestamp_newest(path) 298 elif kind == KIND_AS_MPY: 299 infile = "{}/{}".format(path, script) 300 outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, script[:-3]) 301 ts_infile = get_timestamp(infile) 302 ts_outfile = get_timestamp(outfile, 0) 303 if ts_infile >= ts_outfile: 304 print("MPY", script) 305 mkdir(outfile) 306 res, out = system( 307 [MPY_CROSS] 308 + args.mpy_cross_flags.split() 309 + ["-o", outfile, "-s", script, "-O{}".format(opt), infile] 310 ) 311 if res != 0: 312 print("error compiling {}:".format(infile)) 313 sys.stdout.buffer.write(out) 314 raise SystemExit(1) 315 ts_outfile = get_timestamp(outfile) 316 mpy_files.append(outfile) 317 else: 318 assert kind == KIND_MPY 319 infile = "{}/{}".format(path, script) 320 mpy_files.append(infile) 321 ts_outfile = get_timestamp(infile) 322 ts_newest = max(ts_newest, ts_outfile) 323 324 # Check if output file needs generating 325 if ts_newest < get_timestamp(args.output, 0): 326 # No files are newer than output file so it does not need updating 327 return 328 329 # Freeze paths as strings 330 res, output_str = system([sys.executable, MAKE_FROZEN] + str_paths) 331 if res != 0: 332 print("error freezing strings {}: {}".format(str_paths, output_str)) 333 sys.exit(1) 334 335 # Freeze .mpy files 336 if mpy_files: 337 res, output_mpy = system( 338 [ 339 sys.executable, 340 MPY_TOOL, 341 "-f", 342 "-q", 343 args.build_dir + "/genhdr/qstrdefs.preprocessed.h", 344 ] 345 + args.mpy_tool_flags.split() 346 + mpy_files 347 ) 348 if res != 0: 349 print("error freezing mpy {}:".format(mpy_files)) 350 print(str(output_mpy, "utf8")) 351 sys.exit(1) 352 else: 353 output_mpy = ( 354 b'#include "py/emitglue.h"\n' 355 b"extern const qstr_pool_t mp_qstr_const_pool;\n" 356 b"const qstr_pool_t mp_qstr_frozen_const_pool = {\n" 357 b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, 0\n" 358 b"};\n" 359 b'const char mp_frozen_mpy_names[1] = {"\\0"};\n' 360 b"const mp_raw_code_t *const mp_frozen_mpy_content[1] = {NULL};\n" 361 ) 362 363 # Generate output 364 print("GEN", args.output) 365 mkdir(args.output) 366 with open(args.output, "wb") as f: 367 f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_STR\n//\n") 368 f.write(output_str) 369 f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_MPY\n//\n") 370 f.write(output_mpy) 371 372 373if __name__ == "__main__": 374 main() 375