1#
2# upip - Package manager for MicroPython
3#
4# Copyright (c) 2015-2018 Paul Sokolovsky
5#
6# Licensed under the MIT license.
7#
8import sys
9import gc
10import uos as os
11import uerrno as errno
12import ujson as json
13import uzlib
14import upip_utarfile as tarfile
15
16gc.collect()
17
18
19debug = False
20index_urls = ["https://micropython.org/pi", "https://pypi.org/pypi"]
21install_path = None
22cleanup_files = []
23gzdict_sz = 16 + 15
24
25file_buf = bytearray(512)
26
27
28class NotFoundError(Exception):
29    pass
30
31
32def op_split(path):
33    if path == "":
34        return ("", "")
35    r = path.rsplit("/", 1)
36    if len(r) == 1:
37        return ("", path)
38    head = r[0]
39    if not head:
40        head = "/"
41    return (head, r[1])
42
43
44def op_basename(path):
45    return op_split(path)[1]
46
47
48# Expects *file* name
49def _makedirs(name, mode=0o777):
50    ret = False
51    s = ""
52    comps = name.rstrip("/").split("/")[:-1]
53    if comps[0] == "":
54        s = "/"
55    for c in comps:
56        if s and s[-1] != "/":
57            s += "/"
58        s += c
59        try:
60            os.mkdir(s)
61            ret = True
62        except OSError as e:
63            if e.errno != errno.EEXIST and e.errno != errno.EISDIR:
64                raise e
65            ret = False
66    return ret
67
68
69def save_file(fname, subf):
70    global file_buf
71    with open(fname, "wb") as outf:
72        while True:
73            sz = subf.readinto(file_buf)
74            if not sz:
75                break
76            outf.write(file_buf, sz)
77
78
79def install_tar(f, prefix):
80    meta = {}
81    for info in f:
82        # print(info)
83        fname = info.name
84        try:
85            fname = fname[fname.index("/") + 1 :]
86        except ValueError:
87            fname = ""
88
89        save = True
90        for p in ("setup.", "PKG-INFO", "README"):
91            # print(fname, p)
92            if fname.startswith(p) or ".egg-info" in fname:
93                if fname.endswith("/requires.txt"):
94                    meta["deps"] = f.extractfile(info).read()
95                save = False
96                if debug:
97                    print("Skipping", fname)
98                break
99
100        if save:
101            outfname = prefix + fname
102            if info.type != tarfile.DIRTYPE:
103                if debug:
104                    print("Extracting " + outfname)
105                _makedirs(outfname)
106                subf = f.extractfile(info)
107                save_file(outfname, subf)
108    return meta
109
110
111def expandhome(s):
112    if "~/" in s:
113        h = os.getenv("HOME")
114        s = s.replace("~/", h + "/")
115    return s
116
117
118import ussl
119import usocket
120
121warn_ussl = True
122
123
124def url_open(url):
125    global warn_ussl
126
127    if debug:
128        print(url)
129
130    proto, _, host, urlpath = url.split("/", 3)
131    try:
132        port = 443
133        if ":" in host:
134            host, port = host.split(":")
135            port = int(port)
136        ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM)
137    except OSError as e:
138        fatal("Unable to resolve %s (no Internet?)" % host, e)
139    # print("Address infos:", ai)
140    ai = ai[0]
141
142    s = usocket.socket(ai[0], ai[1], ai[2])
143    try:
144        # print("Connect address:", addr)
145        s.connect(ai[-1])
146
147        if proto == "https:":
148            s = ussl.wrap_socket(s, server_hostname=host)
149            if warn_ussl:
150                print("Warning: %s SSL certificate is not validated" % host)
151                warn_ussl = False
152
153        # MicroPython rawsocket module supports file interface directly
154        s.write("GET /%s HTTP/1.0\r\nHost: %s:%s\r\n\r\n" % (urlpath, host, port))
155        l = s.readline()
156        protover, status, msg = l.split(None, 2)
157        if status != b"200":
158            if status == b"404" or status == b"301":
159                raise NotFoundError("Package not found")
160            raise ValueError(status)
161        while 1:
162            l = s.readline()
163            if not l:
164                raise ValueError("Unexpected EOF in HTTP headers")
165            if l == b"\r\n":
166                break
167    except Exception as e:
168        s.close()
169        raise e
170
171    return s
172
173
174def get_pkg_metadata(name):
175    for url in index_urls:
176        try:
177            f = url_open("%s/%s/json" % (url, name))
178        except NotFoundError:
179            continue
180        try:
181            return json.load(f)
182        finally:
183            f.close()
184    raise NotFoundError("Package not found")
185
186
187def fatal(msg, exc=None):
188    print("Error:", msg)
189    if exc and debug:
190        raise exc
191    sys.exit(1)
192
193
194def install_pkg(pkg_spec, install_path):
195    data = get_pkg_metadata(pkg_spec)
196
197    latest_ver = data["info"]["version"]
198    packages = data["releases"][latest_ver]
199    del data
200    gc.collect()
201    assert len(packages) == 1
202    package_url = packages[0]["url"]
203    print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url))
204    package_fname = op_basename(package_url)
205    f1 = url_open(package_url)
206    try:
207        f2 = uzlib.DecompIO(f1, gzdict_sz)
208        f3 = tarfile.TarFile(fileobj=f2)
209        meta = install_tar(f3, install_path)
210    finally:
211        f1.close()
212    del f3
213    del f2
214    gc.collect()
215    return meta
216
217
218def install(to_install, install_path=None):
219    # Calculate gzip dictionary size to use
220    global gzdict_sz
221    sz = gc.mem_free() + gc.mem_alloc()
222    if sz <= 65536:
223        gzdict_sz = 16 + 12
224
225    if install_path is None:
226        install_path = get_install_path()
227    if install_path[-1] != "/":
228        install_path += "/"
229    if not isinstance(to_install, list):
230        to_install = [to_install]
231    print("Installing to: " + install_path)
232    # sets would be perfect here, but don't depend on them
233    installed = []
234    try:
235        while to_install:
236            if debug:
237                print("Queue:", to_install)
238            pkg_spec = to_install.pop(0)
239            if pkg_spec in installed:
240                continue
241            meta = install_pkg(pkg_spec, install_path)
242            installed.append(pkg_spec)
243            if debug:
244                print(meta)
245            deps = meta.get("deps", "").rstrip()
246            if deps:
247                deps = deps.decode("utf-8").split("\n")
248                to_install.extend(deps)
249    except Exception as e:
250        print(
251            "Error installing '{}': {}, packages may be partially installed".format(pkg_spec, e),
252            file=sys.stderr,
253        )
254
255
256def get_install_path():
257    global install_path
258    if install_path is None:
259        # sys.path[0] is current module's path
260        install_path = sys.path[1]
261    install_path = expandhome(install_path)
262    return install_path
263
264
265def cleanup():
266    for fname in cleanup_files:
267        try:
268            os.unlink(fname)
269        except OSError:
270            print("Warning: Cannot delete " + fname)
271
272
273def help():
274    print(
275        """\
276upip - Simple PyPI package manager for MicroPython
277Usage: micropython -m upip install [-p <path>] <package>... | -r <requirements.txt>
278import upip; upip.install(package_or_list, [<path>])
279
280If <path> is not given, packages will be installed into sys.path[1]
281(can be set from MICROPYPATH environment variable, if current system
282supports that)."""
283    )
284    print("Current value of sys.path[1]:", sys.path[1])
285    print(
286        """\
287
288Note: only MicroPython packages (usually, named micropython-*) are supported
289for installation, upip does not support arbitrary code in setup.py.
290"""
291    )
292
293
294def main():
295    global debug
296    global index_urls
297    global install_path
298    install_path = None
299
300    if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
301        help()
302        return
303
304    if sys.argv[1] != "install":
305        fatal("Only 'install' command supported")
306
307    to_install = []
308
309    i = 2
310    while i < len(sys.argv) and sys.argv[i][0] == "-":
311        opt = sys.argv[i]
312        i += 1
313        if opt == "-h" or opt == "--help":
314            help()
315            return
316        elif opt == "-p":
317            install_path = sys.argv[i]
318            i += 1
319        elif opt == "-r":
320            list_file = sys.argv[i]
321            i += 1
322            with open(list_file) as f:
323                while True:
324                    l = f.readline()
325                    if not l:
326                        break
327                    if l[0] == "#":
328                        continue
329                    to_install.append(l.rstrip())
330        elif opt == "-i":
331            index_urls = [sys.argv[i]]
332            i += 1
333        elif opt == "--debug":
334            debug = True
335        else:
336            fatal("Unknown/unsupported option: " + opt)
337
338    to_install.extend(sys.argv[i:])
339    if not to_install:
340        help()
341        return
342
343    install(to_install)
344
345    if not debug:
346        cleanup()
347
348
349if __name__ == "__main__":
350    main()
351