1#!/usr/bin/env python
2
3# Copyright 2018 The Fuchsia Authors
4#
5# Use of this source code is governed by a MIT-style
6# license that can be found in the LICENSE file or at
7# https://opensource.org/licenses/MIT
8"""
9This tool uses the contents of the abigen-generated syscalls/definitions.json
10to update docs/syscalls/.
11
12It is not run automatically as part of the build for now (to allow confirmation
13of what it does). So it should be run manually after updating syscalls.abigen
14and building zircon, followed by uploading the changes to docs/ as a CL.
15
16It updates the signature, synopsis, and rights annotations, and corrects some
17formatting.
18"""
19
20import argparse
21import json
22import os
23import re
24import subprocess
25import sys
26
27SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
28
29STANDARD_COMMENT = '<!-- Updated by update-docs-from-abigen, do not edit. -->'
30STANDARD_BLOCK_HEADER = ['', STANDARD_COMMENT, '']
31
32REFERENCES_COMMENT = \
33        '<!-- References updated by update-docs-from-abigen, do not edit. -->'
34
35
36def parse_args():
37    parser = argparse.ArgumentParser(
38        description=__doc__,
39        formatter_class=argparse.RawDescriptionHelpFormatter)
40    parser.add_argument(
41        '--json',
42        default=os.path.normpath(
43            os.path.join(SCRIPT_DIR, os.pardir, 'build-x64', 'gen', 'global',
44                         'include', 'zircon', 'syscalls', 'definitions.json')),
45        help='path to abigen .json output')
46    parser.add_argument(
47        '--docroot',
48        default=os.path.normpath(
49            os.path.join(SCRIPT_DIR, os.pardir, 'docs', 'syscalls')),
50        help='root of docs/syscalls/ to be updated')
51    parser.add_argument(
52        '--generate-missing',
53        default=False,
54        action="store_true",
55        help='if set, generate stubs for any syscalls that are missing')
56    parser.add_argument('name', nargs='*', help='only generate these syscalls')
57    return parser.parse_args()
58
59
60def break_into_sentences(stream):
61    """Partition on '.' to break into chunks. '.' can't appear elsewhere
62    in the input stream."""
63    sentences = []
64    cur = []
65    for tok in stream:
66        cur.append(tok)
67        if tok == '.':
68            sentences.append(cur)
69            cur = []
70    assert not cur, cur
71    return sentences
72
73
74def match_sentence_form(sentence, arg_names):
75    """Matches a known sentence form, returning a format string and a dict for
76    substitution. The values in dict are converted to markdown format.
77
78    Certain TERMINALS are special:
79    - ARG must appear in arg_names
80    - RIGHT must be a valid ZX_RIGHT_
81    - TYPE must be a valid ZX_OBJ_TYPE_
82    - RSRC must be a valid ZX_RSRC_KIND_
83
84    VALUE is a generic unchecked value type, used for masks, options, etc.
85    """
86    sentence_forms = [
87        ['None', '.'],
88        ['ARG', 'must', 'have', 'RIGHT', '.'],
89        ['ARG', 'must', 'have', 'resource', 'kind', 'RSRC', '.'],
90        ['ARG', 'must', 'be', 'of', 'type', 'TYPE', '.'],
91        [
92            'ARG', 'must', 'be', 'of', 'type', 'TYPE', 'and', 'have', 'RIGHT1',
93            'and', 'have', 'RIGHT2', '.'
94        ],
95        [
96            'ARG', 'must', 'be', 'of', 'type', 'TYPE', 'and', 'have', 'RIGHT',
97            '.'
98        ],
99        [
100            'ARG', 'must', 'be', 'of', 'type', 'TYPE1', 'or', 'TYPE2', 'and',
101            'have', 'RIGHT', '.'
102        ],
103        [
104            'If', 'ARG1', 'is', 'VALUE', ',', 'ARG2', 'must', 'have', 'RIGHT',
105            '.'
106        ],
107        [
108            'If', 'ARG1', 'is', 'VALUE', ',', 'ARG2', 'must', 'have',
109            'resource', 'kind', 'RSRC', '.'
110        ],
111        [
112            'If', 'ARG1', 'is', 'VALUE', ',', 'ARG2', 'must', 'be', 'of',
113            'type', 'TYPE', '.'
114        ],
115        [
116            'If', 'ARG1', 'is', 'VALUE', ',', 'ARG2', 'must', 'be', 'of',
117            'type', 'TYPE', 'and', 'have', 'RIGHT', '.'
118        ],
119        [
120            'If', 'ARG1', '&', 'VALUE', ',', 'ARG2', 'must', 'be', 'of', 'type',
121            'TYPE', 'and', 'have', 'RIGHT', '.'
122        ],
123        ['Every', 'entry', 'of', 'ARG', 'must', 'have', 'RIGHT', '.'],
124        [
125            'Every', 'entry', 'of', 'ARG', 'must', 'have', 'a',
126            'WAITITEMMEMBER', 'field', 'with', 'RIGHT', '.'
127        ],
128
129        # TODO(ZX-2399) TODO(scottmg): This is a hack specifically for
130        # zx_channel_call_args_t. Trying to make a pseudo-generic case (that
131        # handles the length from wr_num_handles, etc.) for this doesn't seem
132        # worth the trouble at the moment, since it's only checking that the
133        # handles have TRANSFER anyway. Revisit if/when there's more instances
134        # like this.
135        ['All', 'wr_handles', 'of', 'ARG', 'must', 'have', 'RIGHT', '.'],
136    ]
137
138    all_rights = set([
139        'ZX_RIGHT_NONE',
140        'ZX_RIGHT_DUPLICATE',
141        'ZX_RIGHT_TRANSFER',
142        'ZX_RIGHT_READ',
143        'ZX_RIGHT_WRITE',
144        'ZX_RIGHT_EXECUTE',
145        'ZX_RIGHT_MAP',
146        'ZX_RIGHT_GET_PROPERTY',
147        'ZX_RIGHT_SET_PROPERTY',
148        'ZX_RIGHT_ENUMERATE',
149        'ZX_RIGHT_DESTROY',
150        'ZX_RIGHT_SET_POLICY',
151        'ZX_RIGHT_GET_POLICY',
152        'ZX_RIGHT_SIGNAL',
153        'ZX_RIGHT_SIGNAL_PEER',
154        'ZX_RIGHT_WAIT',
155        'ZX_RIGHT_INSPECT',
156        'ZX_RIGHT_MANAGE_JOB',
157        'ZX_RIGHT_MANAGE_PROCESS',
158        'ZX_RIGHT_MANAGE_THREAD',
159        'ZX_RIGHT_APPLY_PROFILE',
160    ])
161
162    all_types = set([
163        'ZX_OBJ_TYPE_PROCESS',
164        'ZX_OBJ_TYPE_THREAD',
165        'ZX_OBJ_TYPE_VMO',
166        'ZX_OBJ_TYPE_CHANNEL',
167        'ZX_OBJ_TYPE_EVENT',
168        'ZX_OBJ_TYPE_PORT',
169        'ZX_OBJ_TYPE_INTERRUPT',
170        'ZX_OBJ_TYPE_PCI_DEVICE',
171        'ZX_OBJ_TYPE_LOG',
172        'ZX_OBJ_TYPE_SOCKET',
173        'ZX_OBJ_TYPE_RESOURCE',
174        'ZX_OBJ_TYPE_EVENTPAIR',
175        'ZX_OBJ_TYPE_JOB',
176        'ZX_OBJ_TYPE_VMAR',
177        'ZX_OBJ_TYPE_FIFO',
178        'ZX_OBJ_TYPE_GUEST',
179        'ZX_OBJ_TYPE_VCPU',
180        'ZX_OBJ_TYPE_TIMER',
181        'ZX_OBJ_TYPE_IOMMU',
182        'ZX_OBJ_TYPE_BTI',
183        'ZX_OBJ_TYPE_PROFILE',
184        'ZX_OBJ_TYPE_PMT',
185        'ZX_OBJ_TYPE_SUSPEND_TOKEN',
186        'ZX_OBJ_TYPE_PAGER',
187    ])
188
189    all_rsrcs = set([
190        'ZX_RSRC_KIND_MMIO',
191        'ZX_RSRC_KIND_IRQ',
192        'ZX_RSRC_KIND_IOPORT',
193        'ZX_RSRC_KIND_HYPERVISOR',
194        'ZX_RSRC_KIND_ROOT',
195        'ZX_RSRC_KIND_VMEX',
196        'ZX_RSRC_KIND_SMC',
197    ])
198
199    # There's only two structs in zircon/types.h, so hardcoding this here is
200    # a bit stinky, but probably OK.
201    members_of_zx_wait_item_t = set([
202        'handle',
203        'waitfor',
204        'pending',
205    ])
206
207    for form in sentence_forms:
208        result_fmt = ''
209        result_values = {}
210        for f, s in zip(form, sentence):
211            # Literal match.
212            if s == f:
213                if f == '.' or f == ',' or f == '->':
214                    result_fmt += f
215                elif f == '[':
216                    result_fmt += '\['
217                else:
218                    result_fmt += ' ' + f
219            elif f.startswith('ARG'):
220                if s not in arg_names:
221                    break
222                else:
223                    result_values[f] = '*' + s + '*'
224                    result_fmt += ' %(' + f + ')s'
225            elif f.startswith('VALUE'):
226                # TODO(scottmg): Worth checking these in some way?
227                result_fmt += ' %(' + f + ')s'
228                result_values[f] = '**' + s + '**'
229            elif f.startswith('RIGHT'):
230                if s not in all_rights:
231                    break
232                result_fmt += ' %(' + f + ')s'
233                result_values[f] = '**' + s + '**'
234            elif f.startswith('RSRC'):
235                if s not in all_rsrcs:
236                    break
237                result_fmt += ' %(' + f + ')s'
238                result_values[f] = '**' + s + '**'
239            elif f.startswith('TYPE'):
240                if s not in all_types:
241                    break
242                result_fmt += ' %(' + f + ')s'
243                result_values[f] = '**' + s + '**'
244            elif f.startswith('WAITITEMMEMBER'):
245                if s not in members_of_zx_wait_item_t:
246                    break
247                result_fmt += ' %(' + f + ')s'
248                result_values[f] = '*' + s + '*'
249            else:
250                break
251        else:
252            if result_fmt[0] == ' ':
253                result_fmt = result_fmt[1:]
254            return result_fmt, result_values
255    else:
256        return None, None
257
258
259def to_markdown(req, arguments, warn):
260    """Parses a few known forms of rules (see match_sentence_forms).
261
262    Converts |req| to formatted markdown.
263    """
264    sentences = break_into_sentences(req)
265
266    if not sentences:
267        rights = ['TODO(ZX-2399)', '']
268    else:
269        rights = []
270    for sentence in sentences:
271        match_fmt, match_values = match_sentence_form(
272            sentence, [x['name'] for x in arguments])
273        if not match_fmt:
274            warn('failed to parse: ' + repr(sentence))
275            raise SystemExit(1)
276        else:
277            rights.append(match_fmt % match_values)
278            rights.append('')
279
280    return STANDARD_BLOCK_HEADER + rights
281
282
283def find_block(lines, name):
284    """Finds a .md block with the given name, and returns (start, end) line
285    indices.
286    """
287    start_index = -1
288    end_index = -1
289    for i, line in enumerate(lines):
290        if line == '## ' + name:
291            start_index = i + 1
292        elif ((start_index >= 0 and line.startswith('## ')) or
293              line == REFERENCES_COMMENT):
294            end_index = i
295            break
296    return start_index, end_index
297
298
299def update_rights(lines, syscall_data, warn):
300    """Updates the RIGHTS block of the .md file in lines.
301    """
302    rights_start_index, rights_end_index = find_block(lines, 'RIGHTS')
303    if rights_start_index == -1 or rights_end_index == -1:
304        warn('did not find RIGHTS section, skipping update')
305        return
306
307    lines[rights_start_index:rights_end_index] = to_markdown(
308        syscall_data['requirements'], syscall_data['arguments'], warn)
309
310
311def make_name_block(syscall_data):
312    start = syscall_data['name'] + ' - '
313    desc = ''
314    for x in syscall_data['top_description']:
315        # TODO(scottmg): This is gross, we should change the abigen parser to
316        # give us a string instead of tokens.
317        if x in (',', '.', '-', '/', '\'', ')'):
318            desc += x
319        else:
320            if desc and desc[-1] not in ('-', '/', '\'', '('):
321                desc += ' '
322            desc += x
323    if not desc:
324        desc = 'TODO(ZX-3106)'
325    return STANDARD_BLOCK_HEADER + [start + desc, '']
326
327
328def update_name(lines, syscall_data, warn):
329    """Updates the NAME block of the .md file in lines.
330    """
331    name_start_index, name_end_index = find_block(lines, 'NAME')
332    if name_start_index == -1 or name_end_index == -1:
333        warn('did not find NAME section, skipping update')
334        return
335
336    lines[name_start_index:name_end_index] = make_name_block(syscall_data)
337
338
339def make_synopsis_block(syscall_data, warn):
340    headers = set([
341        '#include <zircon/syscalls.h>',
342    ])
343    extra_headers = {}
344    for arg in syscall_data['arguments']:
345        if arg['type'] == 'zx_port_packet_t':
346            headers.add('#include <zircon/syscalls/port.h>')
347        elif (arg['type'] == 'zx_smc_parameters_t' or
348              arg['type'] == 'zx_smc_result_t'):
349            headers.add('#include <zircon/syscalls/smc.h>')
350    header = ['```'] + sorted(list(headers)) + ['']
351
352    def format_arg(x):
353        ret = ''
354        if 'IN' in x['attributes']:
355            ret += 'const '
356        if x['type'] == 'any':
357            ret += 'void '
358        else:
359            ret += x['type'] + ' '
360        if x['is_array']:
361            ret += ' * '
362        ret += ' ' + x['name']
363        return ret
364
365    no_return = ''
366    if 'noreturn' in syscall_data['attributes']: no_return = '[[noreturn]]'
367
368    to_format = (no_return + syscall_data['return_type'] + ' zx_' +
369                 syscall_data['name'] + '(')
370    args = ','.join(format_arg(x) for x in syscall_data['arguments'])
371    if not args:
372        args = 'void'
373    to_format += args + ');'
374
375    CLANG_FORMAT_PATH = os.path.join(SCRIPT_DIR, os.pardir, 'prebuilt',
376                                     'downloads', 'clang', 'bin',
377                                     'clang-format')
378    clang_format = subprocess.Popen([
379        CLANG_FORMAT_PATH,
380        '-style={BasedOnStyle: Google, BinPackParameters: false}'
381    ],
382                                    stdin=subprocess.PIPE,
383                                    stdout=subprocess.PIPE)
384    formatted, _ = clang_format.communicate(to_format)
385    if clang_format.returncode != 0:
386        warn('formatting synopsis failed, skipping update')
387        return None
388
389    footer = [
390        '```',
391        '',
392    ]
393    return STANDARD_BLOCK_HEADER + header + [formatted] + footer
394
395
396def update_synopsis(lines, syscall_data, warn):
397    """Updates the SYNOPSIS block of the .md file in lines.
398    """
399    start_index, end_index = find_block(lines, 'SYNOPSIS')
400    if start_index == -1 or end_index == -1:
401        warn('did not find SYNOPSIS section, skipping update')
402        return
403
404    syn = make_synopsis_block(syscall_data, warn)
405    if not syn:
406        return
407    lines[start_index:end_index] = syn
408
409
410def update_title(lines, syscall_data, _):
411    """Updates the main title of the .md file given by |filename|.
412    """
413    correct_title = '# zx_' + syscall_data['name']
414    if lines[0] != correct_title:
415        lines[0] = correct_title
416
417
418def generate_stub(md):
419    """Makes a mostly-empty file that can then be filled out by later update
420    functions."""
421
422    stub = '''\
423# zx_xyz
424
425## NAME
426
427## SYNOPSIS
428
429## DESCRIPTION
430
431TODO(ZX-3106)
432
433## RIGHTS
434
435## RETURN VALUE
436
437TODO(ZX-3106)
438
439## ERRORS
440
441TODO(ZX-3106)
442
443## SEE ALSO
444
445TODO(ZX-3106)
446'''
447    with open(md, 'wb') as f:
448        f.write(stub)
449
450
451def check_for_orphans(syscalls, root):
452    """Checks for any .md files that have been orphaned (no longer have an
453    associated abigen entry.)
454    """
455    orphan_count = 0
456    names = set([x['name'] for x in syscalls])
457    for md in os.listdir(root):
458        if not md.endswith('.md'):
459            print >> sys.stderr, 'warning: non .md file %s' % md
460        name = md[:-3]
461        if name not in names:
462            orphan_count += 1
463            print >> sys.stderr, 'warning: %s has no entry in syscalls' % md
464    return orphan_count
465
466
467# A few concept docs that are linked in SEE ALSO sections.
468SEE_ALSO_CONCEPTS = {
469    'rights': '../rights.md',
470    'exceptions': '../exceptions.md',
471    'futex objects': '../objects/futex.md'
472}
473
474def make_see_also_block(referenced_syscalls, concepts, extra):
475    """Makes a formatted SEE ALSO block given a list of syscall names.
476    """
477    result = []
478
479    for concept in sorted(concepts):
480        path = SEE_ALSO_CONCEPTS[concept]
481        result.append(' - [' + concept + '](' + path + ')')
482
483    for sc in sorted(referenced_syscalls):
484        # References to these will be done later by update_syscall_references().
485        result.append(' - [`zx_' + sc + '()`]')
486
487    if extra:
488        extra += ['']
489
490    # No comment header here, because people are still editing this by hand,
491    # we're only canonicalizing it.
492    return [''] + extra + result + ['']
493
494
495def update_seealso(lines, syscall, all_syscall_names, warn):
496    """Rewrites 'SEE ALSO' block to canonical format.
497    """
498    start_index, end_index = find_block(lines, 'SEE ALSO')
499    if start_index == -1:
500        return
501
502    referenced = set()
503    concepts = set()
504    extra = []
505    for line in lines[start_index:end_index]:
506        if not line or line == STANDARD_COMMENT:
507            continue
508
509        handled = False
510        for concept in SEE_ALSO_CONCEPTS:
511            if '[' + concept + ']' in line:
512                concepts.add(concept)
513                handled = True
514        if handled:
515            continue
516
517        for sc in all_syscall_names:
518            old = '[' + sc + ']'
519            new = '[`zx_' + sc + '()`]'
520            if old in line or new in line:
521                referenced.add(sc)
522                break
523        else:
524            warn('unrecognized "see also", keeping before syscalls: ' + line)
525            extra.append(line)
526
527    lines[start_index:end_index] = make_see_also_block(referenced, concepts,
528                                                       extra)
529
530
531SYSCALL_RE = {}
532
533
534def update_syscall_references(lines, syscall, all_syscall_names, warn):
535    """Attempts to update all syscall references to a canonical format, and
536    linkifies them to their corresponding syscall.
537
538    TODO(ZX-3106): It'd be nice to do the references from outside of
539    docs/syscalls/ into syscalls too, in a similar style.
540    """
541
542    text = '\n'.join(lines)
543
544    # Precompile these regexes as it takes measurable time.
545    if not SYSCALL_RE:
546        for sc in all_syscall_names:
547            # Look for **zx_stuff()** and [`zx_stuff()`], with both "zx_" and
548            # the () being optional.
549            SYSCALL_RE[sc] = re.compile(
550                r'\*{2}(?:zx_)?' + sc + r'(?:\(\))?\*{2}(?:\(\))?'
551                r'|'
552                r'(?:\[)`(?:zx_)?' + sc + r'(?:\(\))?`(?:\])?(\(\))?')
553
554    referred_to = set()
555    for sc in all_syscall_names:
556        scre = SYSCALL_RE[sc]
557        self = sc == syscall['name']
558        repl = '`zx_' + sc + '()`'
559        # Don't link to ourselves.
560        if not self:
561            repl = '[' + repl + ']'
562        text, count = scre.subn(repl, text)
563        if count and not self:
564            referred_to.add(sc)
565
566    lines[:] = text.splitlines()
567
568    if REFERENCES_COMMENT not in lines:
569        lines.extend(['', REFERENCES_COMMENT])
570    start_index = lines.index(REFERENCES_COMMENT)
571
572    references = []
573    for ref in sorted(referred_to):
574        references.append('[`zx_' + ref + '()`]: ' + ref + '.md')
575    lines[start_index:] = [REFERENCES_COMMENT, ''] + references
576
577    # Drop references section if it's empty to not be noisy.
578    if lines[-3:] == ['', REFERENCES_COMMENT, '']:
579        lines[:] = lines[:-3]
580
581
582def main():
583    args = parse_args()
584    inf = os.path.relpath(args.json)
585    outf = os.path.relpath(args.docroot)
586    print 'using %s as input and updating %s...' % (inf, outf)
587    data = json.loads(open(inf, 'rb').read())
588    missing_count = 0
589    all_syscall_names = set(x['name'] for x in data['syscalls'])
590    for syscall in data['syscalls']:
591        name = syscall['name']
592        if args.name and name not in args.name:
593            continue
594        md = os.path.join(outf, name + '.md')
595
596        if not os.path.exists(md) and args.generate_missing:
597            generate_stub(md)
598
599        if not os.path.exists(md):
600            print >> sys.stderr, (
601                'warning: %s not found for updating, skipping update' % md)
602            missing_count += 1
603        else:
604            with open(md, 'rb') as f:
605                lines = f.read().splitlines()
606
607            assert (lines)
608
609            def warn(msg):
610                print >> sys.stderr, 'warning: %s: %s' % (md, msg)
611
612            update_title(lines, syscall, warn)
613            update_name(lines, syscall, warn)
614            update_synopsis(lines, syscall, warn)
615            update_rights(lines, syscall, warn)
616            update_seealso(lines, syscall, all_syscall_names, warn)
617            update_syscall_references(lines, syscall, all_syscall_names, warn)
618
619            with open(md, 'wb') as f:
620                f.write('\n'.join(lines) + '\n')
621
622    if missing_count > 0:
623        print >> sys.stderr, 'warning: %d missing .md files' % missing_count
624    missing_count += check_for_orphans(data['syscalls'], outf)
625    return missing_count
626
627
628if __name__ == '__main__':
629    sys.exit(main())
630