1#!/usr/bin/env python3
2#
3# This file is part of the MicroPython project, http://micropython.org/
4#
5# The MIT License (MIT)
6#
7# Copyright (c) 2019 Damien P. George
8#
9# Permission is hereby granted, free of charge, to any person obtaining a copy
10# of this software and associated documentation files (the "Software"), to deal
11# in the Software without restriction, including without limitation the rights
12# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13# copies of the Software, and to permit persons to whom the Software is
14# furnished to do so, subject to the following conditions:
15#
16# The above copyright notice and this permission notice shall be included in
17# all copies or substantial portions of the Software.
18#
19# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25# THE SOFTWARE.
26
27from __future__ import print_function
28import sys
29import os
30import subprocess
31
32
33###########################################################################
34# Public functions to be used in the manifest
35
36
37def include(manifest, **kwargs):
38    """Include another manifest.
39
40    The manifest argument can be a string (filename) or an iterable of
41    strings.
42
43    Relative paths are resolved with respect to the current manifest file.
44
45    Optional kwargs can be provided which will be available to the
46    included script via the `options` variable.
47
48    e.g. include("path.py", extra_features=True)
49
50    in path.py:
51        options.defaults(standard_features=True)
52
53        # freeze minimal modules.
54        if options.standard_features:
55            # freeze standard modules.
56        if options.extra_features:
57            # freeze extra modules.
58    """
59
60    if not isinstance(manifest, str):
61        for m in manifest:
62            include(m)
63    else:
64        manifest = convert_path(manifest)
65        with open(manifest) as f:
66            # Make paths relative to this manifest file while processing it.
67            # Applies to includes and input files.
68            prev_cwd = os.getcwd()
69            os.chdir(os.path.dirname(manifest))
70            exec(f.read(), globals(), {"options": IncludeOptions(**kwargs)})
71            os.chdir(prev_cwd)
72
73
74def freeze(path, script=None, opt=0):
75    """Freeze the input, automatically determining its type.  A .py script
76    will be compiled to a .mpy first then frozen, and a .mpy file will be
77    frozen directly.
78
79    `path` must be a directory, which is the base directory to search for
80    files from.  When importing the resulting frozen modules, the name of
81    the module will start after `path`, ie `path` is excluded from the
82    module name.
83
84    If `path` is relative, it is resolved to the current manifest.py.
85    Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need
86    to access specific paths.
87
88    If `script` is None all files in `path` will be frozen.
89
90    If `script` is an iterable then freeze() is called on all items of the
91    iterable (with the same `path` and `opt` passed through).
92
93    If `script` is a string then it specifies the file or directory to
94    freeze, and can include extra directories before the file or last
95    directory.  The file or directory will be searched for in `path`.  If
96    `script` is a directory then all files in that directory will be frozen.
97
98    `opt` is the optimisation level to pass to mpy-cross when compiling .py
99    to .mpy.
100    """
101
102    freeze_internal(KIND_AUTO, path, script, opt)
103
104
105def freeze_as_str(path):
106    """Freeze the given `path` and all .py scripts within it as a string,
107    which will be compiled upon import.
108    """
109
110    freeze_internal(KIND_AS_STR, path, None, 0)
111
112
113def freeze_as_mpy(path, script=None, opt=0):
114    """Freeze the input (see above) by first compiling the .py scripts to
115    .mpy files, then freezing the resulting .mpy files.
116    """
117
118    freeze_internal(KIND_AS_MPY, path, script, opt)
119
120
121def freeze_mpy(path, script=None, opt=0):
122    """Freeze the input (see above), which must be .mpy files that are
123    frozen directly.
124    """
125
126    freeze_internal(KIND_MPY, path, script, opt)
127
128
129###########################################################################
130# Internal implementation
131
132KIND_AUTO = 0
133KIND_AS_STR = 1
134KIND_AS_MPY = 2
135KIND_MPY = 3
136
137VARS = {}
138
139manifest_list = []
140
141
142class IncludeOptions:
143    def __init__(self, **kwargs):
144        self._kwargs = kwargs
145        self._defaults = {}
146
147    def defaults(self, **kwargs):
148        self._defaults = kwargs
149
150    def __getattr__(self, name):
151        return self._kwargs.get(name, self._defaults.get(name, None))
152
153
154class FreezeError(Exception):
155    pass
156
157
158def system(cmd):
159    try:
160        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
161        return 0, output
162    except subprocess.CalledProcessError as er:
163        return -1, er.output
164
165
166def convert_path(path):
167    # Perform variable substituion.
168    for name, value in VARS.items():
169        path = path.replace("$({})".format(name), value)
170    # Convert to absolute path (so that future operations don't rely on
171    # still being chdir'ed).
172    return os.path.abspath(path)
173
174
175def get_timestamp(path, default=None):
176    try:
177        stat = os.stat(path)
178        return stat.st_mtime
179    except OSError:
180        if default is None:
181            raise FreezeError("cannot stat {}".format(path))
182        return default
183
184
185def get_timestamp_newest(path):
186    ts_newest = 0
187    for dirpath, dirnames, filenames in os.walk(path, followlinks=True):
188        for f in filenames:
189            ts_newest = max(ts_newest, get_timestamp(os.path.join(dirpath, f)))
190    return ts_newest
191
192
193def mkdir(filename):
194    path = os.path.dirname(filename)
195    if not os.path.isdir(path):
196        os.makedirs(path)
197
198
199def freeze_internal(kind, path, script, opt):
200    path = convert_path(path)
201    if not os.path.isdir(path):
202        raise FreezeError("freeze path must be a directory: {}".format(path))
203    if script is None and kind == KIND_AS_STR:
204        if any(f[0] == KIND_AS_STR for f in manifest_list):
205            raise FreezeError("can only freeze one str directory")
206        manifest_list.append((KIND_AS_STR, path, script, opt))
207    elif script is None or isinstance(script, str) and script.find(".") == -1:
208        # Recursively search `path` for files to freeze, optionally restricted
209        # to a subdirectory specified by `script`
210        if script is None:
211            subdir = ""
212        else:
213            subdir = "/" + script
214        for dirpath, dirnames, filenames in os.walk(path + subdir, followlinks=True):
215            for f in filenames:
216                freeze_internal(kind, path, (dirpath + "/" + f)[len(path) + 1 :], opt)
217    elif not isinstance(script, str):
218        # `script` is an iterable of items to freeze
219        for s in script:
220            freeze_internal(kind, path, s, opt)
221    else:
222        # `script` should specify an individual file to be frozen
223        extension_kind = {KIND_AS_MPY: ".py", KIND_MPY: ".mpy"}
224        if kind == KIND_AUTO:
225            for k, ext in extension_kind.items():
226                if script.endswith(ext):
227                    kind = k
228                    break
229            else:
230                print("warn: unsupported file type, skipped freeze: {}".format(script))
231                return
232        wanted_extension = extension_kind[kind]
233        if not script.endswith(wanted_extension):
234            raise FreezeError("expecting a {} file, got {}".format(wanted_extension, script))
235        manifest_list.append((kind, path, script, opt))
236
237
238def main():
239    # Parse arguments
240    import argparse
241
242    cmd_parser = argparse.ArgumentParser(
243        description="A tool to generate frozen content in MicroPython firmware images."
244    )
245    cmd_parser.add_argument("-o", "--output", help="output path")
246    cmd_parser.add_argument("-b", "--build-dir", help="output path")
247    cmd_parser.add_argument(
248        "-f", "--mpy-cross-flags", default="", help="flags to pass to mpy-cross"
249    )
250    cmd_parser.add_argument("-v", "--var", action="append", help="variables to substitute")
251    cmd_parser.add_argument("--mpy-tool-flags", default="", help="flags to pass to mpy-tool")
252    cmd_parser.add_argument("files", nargs="+", help="input manifest list")
253    args = cmd_parser.parse_args()
254
255    # Extract variables for substitution.
256    for var in args.var:
257        name, value = var.split("=", 1)
258        if os.path.exists(value):
259            value = os.path.abspath(value)
260        VARS[name] = value
261
262    if "MPY_DIR" not in VARS or "PORT_DIR" not in VARS:
263        print("MPY_DIR and PORT_DIR variables must be specified")
264        sys.exit(1)
265
266    # Get paths to tools
267    MAKE_FROZEN = VARS["MPY_DIR"] + "/tools/make-frozen.py"
268    MPY_CROSS = VARS["MPY_DIR"] + "/mpy-cross/mpy-cross"
269    if sys.platform == "win32":
270        MPY_CROSS += ".exe"
271    MPY_CROSS = os.getenv("MICROPY_MPYCROSS", MPY_CROSS)
272    MPY_TOOL = VARS["MPY_DIR"] + "/tools/mpy-tool.py"
273
274    # Ensure mpy-cross is built
275    if not os.path.exists(MPY_CROSS):
276        print("mpy-cross not found at {}, please build it first".format(MPY_CROSS))
277        sys.exit(1)
278
279    # Include top-level inputs, to generate the manifest
280    for input_manifest in args.files:
281        try:
282            if input_manifest.endswith(".py"):
283                include(input_manifest)
284            else:
285                exec(input_manifest)
286        except FreezeError as er:
287            print('freeze error executing "{}": {}'.format(input_manifest, er.args[0]))
288            sys.exit(1)
289
290    # Process the manifest
291    str_paths = []
292    mpy_files = []
293    ts_newest = 0
294    for kind, path, script, opt in manifest_list:
295        if kind == KIND_AS_STR:
296            str_paths.append(path)
297            ts_outfile = get_timestamp_newest(path)
298        elif kind == KIND_AS_MPY:
299            infile = "{}/{}".format(path, script)
300            outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, script[:-3])
301            ts_infile = get_timestamp(infile)
302            ts_outfile = get_timestamp(outfile, 0)
303            if ts_infile >= ts_outfile:
304                print("MPY", script)
305                mkdir(outfile)
306                res, out = system(
307                    [MPY_CROSS]
308                    + args.mpy_cross_flags.split()
309                    + ["-o", outfile, "-s", script, "-O{}".format(opt), infile]
310                )
311                if res != 0:
312                    print("error compiling {}:".format(infile))
313                    sys.stdout.buffer.write(out)
314                    raise SystemExit(1)
315                ts_outfile = get_timestamp(outfile)
316            mpy_files.append(outfile)
317        else:
318            assert kind == KIND_MPY
319            infile = "{}/{}".format(path, script)
320            mpy_files.append(infile)
321            ts_outfile = get_timestamp(infile)
322        ts_newest = max(ts_newest, ts_outfile)
323
324    # Check if output file needs generating
325    if ts_newest < get_timestamp(args.output, 0):
326        # No files are newer than output file so it does not need updating
327        return
328
329    # Freeze paths as strings
330    res, output_str = system([sys.executable, MAKE_FROZEN] + str_paths)
331    if res != 0:
332        print("error freezing strings {}: {}".format(str_paths, output_str))
333        sys.exit(1)
334
335    # Freeze .mpy files
336    if mpy_files:
337        res, output_mpy = system(
338            [
339                sys.executable,
340                MPY_TOOL,
341                "-f",
342                "-q",
343                args.build_dir + "/genhdr/qstrdefs.preprocessed.h",
344            ]
345            + args.mpy_tool_flags.split()
346            + mpy_files
347        )
348        if res != 0:
349            print("error freezing mpy {}:".format(mpy_files))
350            print(str(output_mpy, "utf8"))
351            sys.exit(1)
352    else:
353        output_mpy = (
354            b'#include "py/emitglue.h"\n'
355            b"extern const qstr_pool_t mp_qstr_const_pool;\n"
356            b"const qstr_pool_t mp_qstr_frozen_const_pool = {\n"
357            b"    (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, 0\n"
358            b"};\n"
359            b'const char mp_frozen_mpy_names[1] = {"\\0"};\n'
360            b"const mp_raw_code_t *const mp_frozen_mpy_content[1] = {NULL};\n"
361        )
362
363    # Generate output
364    print("GEN", args.output)
365    mkdir(args.output)
366    with open(args.output, "wb") as f:
367        f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_STR\n//\n")
368        f.write(output_str)
369        f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_MPY\n//\n")
370        f.write(output_mpy)
371
372
373if __name__ == "__main__":
374    main()
375