1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4import sys, os, re
5
6class Fail(Exception):
7    pass
8
9class State(object):
10
11    def __init__(self, input, output):
12
13        self.source = input
14        self.input  = open_file_or_fd(input, "r", 2)
15        self.output = open_file_or_fd(output, "w", 2)
16
17        # State parsed from input
18        self.names = {} # Name => value mapping
19        self.raw_special = set()
20        self.raw_pv = set()
21        self.raw_hvm_shadow = set()
22        self.raw_hvm_hap = set()
23
24        # State calculated
25        self.nr_entries = 0 # Number of words in a featureset
26        self.common_1d = 0 # Common features between 1d and e1d
27        self.known = [] # All known features
28        self.special = [] # Features with special semantics
29        self.pv = []
30        self.hvm_shadow = []
31        self.hvm_hap = []
32        self.bitfields = [] # Text to declare named bitfields in C
33
34def parse_definitions(state):
35    """
36    Parse featureset information from @param f and mutate the global
37    namespace with symbols
38    """
39    feat_regex = re.compile(
40        r"^XEN_CPUFEATURE\(([A-Z0-9_]+),"
41        "\s+([\s\d]+\*[\s\d]+\+[\s\d]+)\)"
42        "\s+/\*([\w!]*) .*$")
43
44    this = sys.modules[__name__]
45
46    for l in state.input.readlines():
47        # Short circuit the regex...
48        if not l.startswith("XEN_CPUFEATURE("):
49            continue
50
51        res = feat_regex.match(l)
52
53        if res is None:
54            raise Fail("Failed to interpret '%s'" % (l.strip(), ))
55
56        name = res.groups()[0]
57        val = eval(res.groups()[1]) # Regex confines this to a very simple expression
58        attr = res.groups()[2]
59
60        if hasattr(this, name):
61            raise Fail("Duplicate symbol %s" % (name,))
62
63        if val in state.names:
64            raise Fail("Aliased value between %s and %s" %
65                       (name, state.names[val]))
66
67        # Mutate the current namespace to insert a feature literal with its
68        # bit index.  Prepend an underscore if the name starts with a digit.
69        if name[0] in "0123456789":
70            this_name = "_" + name
71        else:
72            this_name = name
73        setattr(this, this_name, val)
74
75        # Construct a reverse mapping of value to name
76        state.names[val] = name
77
78        for a in attr:
79
80            if a == "!":
81                state.raw_special.add(val)
82            elif a in "ASH":
83                if a == "A":
84                    state.raw_pv.add(val)
85                    state.raw_hvm_shadow.add(val)
86                    state.raw_hvm_hap.add(val)
87                elif attr == "S":
88                    state.raw_hvm_shadow.add(val)
89                    state.raw_hvm_hap.add(val)
90                elif attr == "H":
91                    state.raw_hvm_hap.add(val)
92            else:
93                raise Fail("Unrecognised attribute '%s' for %s" % (a, name))
94
95    if len(state.names) == 0:
96        raise Fail("No features found")
97
98def featureset_to_uint32s(fs, nr):
99    """ Represent a featureset as a list of C-compatible uint32_t's """
100
101    bitmap = 0L
102    for f in fs:
103        bitmap |= 1L << f
104
105    words = []
106    while bitmap:
107        words.append(bitmap & ((1L << 32) - 1))
108        bitmap >>= 32
109
110    assert len(words) <= nr
111
112    if len(words) < nr:
113        words.extend([0] * (nr - len(words)))
114
115    return [ "0x%08xU" % x for x in words ]
116
117def format_uint32s(words, indent):
118    """ Format a list of uint32_t's suitable for a macro definition """
119    spaces = " " * indent
120    return spaces + (", \\\n" + spaces).join(words) + ", \\"
121
122
123def crunch_numbers(state):
124
125    # Size of bitmaps
126    state.nr_entries = nr_entries = (max(state.names.keys()) >> 5) + 1
127
128    # Features common between 1d and e1d.
129    common_1d = (FPU, VME, DE, PSE, TSC, MSR, PAE, MCE, CX8, APIC,
130                 MTRR, PGE, MCA, CMOV, PAT, PSE36, MMX, FXSR)
131
132    state.known = featureset_to_uint32s(state.names.keys(), nr_entries)
133    state.common_1d = featureset_to_uint32s(common_1d, 1)[0]
134    state.special = featureset_to_uint32s(state.raw_special, nr_entries)
135    state.pv = featureset_to_uint32s(state.raw_pv, nr_entries)
136    state.hvm_shadow = featureset_to_uint32s(state.raw_hvm_shadow, nr_entries)
137    state.hvm_hap = featureset_to_uint32s(state.raw_hvm_hap, nr_entries)
138
139    #
140    # Feature dependency information.
141    #
142    # !!! WARNING !!!
143    #
144    # A lot of this information is derived from the written text of vendors
145    # software manuals, rather than directly from a statement.  As such, it
146    # does contain guesswork and assumptions, and may not accurately match
147    # hardware implementations.
148    #
149    # It is however designed to create an end result for a guest which does
150    # plausibly match real hardware.
151    #
152    # !!! WARNING !!!
153    #
154    # The format of this dictionary is that the feature in the key is a direct
155    # prerequisite of each feature in the value.
156    #
157    # The first consideration is about which functionality is physically built
158    # on top of other features.  The second consideration, which is more
159    # subjective, is whether real hardware would ever be found supporting
160    # feature X but not Y.
161    #
162    deps = {
163        # FPU is taken to mean support for the x87 regisers as well as the
164        # instructions.  MMX is documented to alias the %MM registers over the
165        # x87 %ST registers in hardware.
166        FPU: [MMX],
167
168        # The PSE36 feature indicates that reserved bits in a PSE superpage
169        # may be used as extra physical address bits.
170        PSE: [PSE36],
171
172        # Entering Long Mode requires that %CR4.PAE is set.  The NX pagetable
173        # bit is only representable in the 64bit PTE format offered by PAE.
174        PAE: [LM, NX],
175
176        TSC: [TSC_DEADLINE, RDTSCP, TSC_ADJUST, ITSC],
177
178        # APIC is special, but X2APIC does depend on APIC being available in
179        # the first place.
180        APIC: [X2APIC, TSC_DEADLINE, EXTAPIC],
181
182        # AMD built MMXExtentions and 3DNow as extentions to MMX.
183        MMX: [MMXEXT, _3DNOW],
184
185        # The FXSAVE/FXRSTOR instructions were introduced into hardware before
186        # SSE, which is why they behave differently based on %CR4.OSFXSAVE and
187        # have their own feature bit.  AMD however introduce the Fast FXSR
188        # feature as an optimisation.
189        FXSR: [FFXSR, SSE],
190
191        # SSE is taken to mean support for the %XMM registers as well as the
192        # instructions.  Several futher instruction sets are built on core
193        # %XMM support, without specific inter-dependencies.  Additionally
194        # AMD has a special mis-alignment sub-mode.
195        SSE: [SSE2, SSE3, SSSE3, SSE4A, MISALIGNSSE,
196              AESNI, SHA],
197
198        # SSE2 was re-specified as core instructions for 64bit.
199        SSE2: [LM],
200
201        # SSE4.1 explicitly depends on SSE3 and SSSE3
202        SSE3: [SSE4_1],
203        SSSE3: [SSE4_1],
204
205        # SSE4.2 explicitly depends on SSE4.1
206        SSE4_1: [SSE4_2],
207
208        # AMD specify no relationship between POPCNT and SSE4.2.  Intel
209        # document that SSE4.2 should be checked for before checking for
210        # POPCNT.  However, it has its own feature bit, and operates on GPRs
211        # rather than %XMM state, so doesn't inherently depend on SSE.
212        # Therefore, we do not specify a dependency between SSE4_2 and POPCNT.
213        #
214        # SSE4_2: [POPCNT]
215
216        # The INVPCID instruction depends on PCID infrastructure being
217        # available.
218        PCID: [INVPCID],
219
220        # XSAVE is an extra set of instructions for state management, but
221        # doesn't constitue new state itself.  Some of the dependent features
222        # are instructions built on top of base XSAVE, while others are new
223        # instruction groups which are specified to require XSAVE for state
224        # management.
225        XSAVE: [XSAVEOPT, XSAVEC, XGETBV1, XSAVES,
226                AVX, MPX, PKU, LWP],
227
228        # AVX is taken to mean hardware support for 256bit registers (which in
229        # practice depends on the VEX prefix to encode), and the instructions
230        # themselves.
231        #
232        # AVX is not taken to mean support for the VEX prefix itself (nor XOP
233        # for the XOP prefix).  VEX/XOP-encoded GPR instructions, such as
234        # those from the BMI{1,2}, TBM and LWP sets function fine in the
235        # absence of any enabled xstate.
236        AVX: [FMA, FMA4, F16C, AVX2, XOP],
237
238        # CX16 is only encodable in Long Mode.  LAHF_LM indicates that the
239        # SAHF/LAHF instructions are reintroduced in Long Mode.  1GB
240        # superpages, PCID and PKU are only available in 4 level paging.
241        LM: [CX16, PCID, LAHF_LM, PAGE1GB, PKU],
242
243        # AMD K6-2+ and K6-III processors shipped with 3DNow+, beyond the
244        # standard 3DNow in the earlier K6 processors.
245        _3DNOW: [_3DNOWEXT],
246
247        # This is just the dependency between AVX512 and AVX2 of XSTATE
248        # feature flags.  If want to use AVX512, AVX2 must be supported and
249        # enabled.
250        AVX2: [AVX512F],
251
252        # AVX512F is taken to mean hardware support for 512bit registers
253        # (which in practice depends on the EVEX prefix to encode), and the
254        # instructions themselves. All further AVX512 features are built on
255        # top of AVX512F
256        AVX512F: [AVX512DQ, AVX512IFMA, AVX512PF, AVX512ER, AVX512CD,
257                  AVX512BW, AVX512VL, AVX512VBMI, AVX512_4VNNIW,
258                  AVX512_4FMAPS, AVX512_VPOPCNTDQ],
259    }
260
261    deep_features = tuple(sorted(deps.keys()))
262    state.deep_deps = {}
263
264    for feat in deep_features:
265
266        seen = [feat]
267        to_process = list(deps[feat])
268
269        while len(to_process):
270
271            # To debug, uncomment the following lines:
272            # def repl(l):
273            #     return "[" + ", ".join((state.names[x] for x in l)) + "]"
274            # print >>sys.stderr, "Feature %s, seen %s, to_process %s " % \
275            #     (state.names[feat], repl(seen), repl(to_process))
276
277            f = to_process.pop(0)
278
279            if f in seen:
280                raise Fail("ERROR: Cycle found with %s when processing %s"
281                           % (state.names[f], state.names[feat]))
282
283            seen.append(f)
284            to_process = list(set(to_process + deps.get(f, [])))
285
286        state.deep_deps[feat] = seen[1:]
287
288    state.deep_features = featureset_to_uint32s(deps.keys(), nr_entries)
289    state.nr_deep_deps = len(state.deep_deps.keys())
290
291    for k, v in state.deep_deps.iteritems():
292        state.deep_deps[k] = featureset_to_uint32s(v, nr_entries)
293
294    # Calculate the bitfield name declarations
295    for word in xrange(nr_entries):
296
297        names = []
298        for bit in xrange(32):
299
300            name = state.names.get(word * 32 + bit, "")
301
302            # Prepend an underscore if the name starts with a digit.
303            if name and name[0] in "0123456789":
304                name = "_" + name
305
306            # Don't generate names for features fast-forwarded from other
307            # state
308            if name in ("APIC", "OSXSAVE", "OSPKE"):
309                name = ""
310
311            names.append(name.lower())
312
313        state.bitfields.append("bool " + ":1, ".join(names) + ":1")
314
315
316def write_results(state):
317    state.output.write(
318"""/*
319 * Automatically generated by %s - Do not edit!
320 * Source data: %s
321 */
322#ifndef __XEN_X86__FEATURESET_DATA__
323#define __XEN_X86__FEATURESET_DATA__
324""" % (sys.argv[0], state.source))
325
326    state.output.write(
327"""
328#define FEATURESET_NR_ENTRIES %s
329
330#define CPUID_COMMON_1D_FEATURES %s
331
332#define INIT_KNOWN_FEATURES { \\\n%s\n}
333
334#define INIT_SPECIAL_FEATURES { \\\n%s\n}
335
336#define INIT_PV_FEATURES { \\\n%s\n}
337
338#define INIT_HVM_SHADOW_FEATURES { \\\n%s\n}
339
340#define INIT_HVM_HAP_FEATURES { \\\n%s\n}
341
342#define NR_DEEP_DEPS %sU
343
344#define INIT_DEEP_FEATURES { \\\n%s\n}
345
346#define INIT_DEEP_DEPS { \\
347""" % (state.nr_entries,
348       state.common_1d,
349       format_uint32s(state.known, 4),
350       format_uint32s(state.special, 4),
351       format_uint32s(state.pv, 4),
352       format_uint32s(state.hvm_shadow, 4),
353       format_uint32s(state.hvm_hap, 4),
354       state.nr_deep_deps,
355       format_uint32s(state.deep_features, 4),
356       ))
357
358    for dep in sorted(state.deep_deps.keys()):
359        state.output.write(
360            "    { %#xU, /* %s */ { \\\n%s\n    }, }, \\\n"
361            % (dep, state.names[dep],
362               format_uint32s(state.deep_deps[dep], 8)
363           ))
364
365    state.output.write(
366"""}
367
368""")
369
370    for idx, text in enumerate(state.bitfields):
371        state.output.write(
372            "#define CPUID_BITFIELD_%d \\\n    %s\n\n"
373            % (idx, text))
374
375    state.output.write(
376"""
377#endif /* __XEN_X86__FEATURESET_DATA__ */
378""")
379
380
381def open_file_or_fd(val, mode, buffering):
382    """
383    If 'val' looks like a decimal integer, open it as an fd.  If not, try to
384    open it as a regular file.
385    """
386
387    fd = -1
388    try:
389        # Does it look like an integer?
390        try:
391            fd = int(val, 10)
392        except ValueError:
393            pass
394
395        if fd == 0:
396            return sys.stdin
397        elif fd == 1:
398            return sys.stdout
399        elif fd == 2:
400            return sys.stderr
401
402        # Try to open it...
403        if fd != -1:
404            return os.fdopen(fd, mode, buffering)
405        else:
406            return open(val, mode, buffering)
407
408    except StandardError, e:
409        if fd != -1:
410            raise Fail("Unable to open fd %d: %s: %s" %
411                       (fd, e.__class__.__name__, e))
412        else:
413            raise Fail("Unable to open file '%s': %s: %s" %
414                       (val, e.__class__.__name__, e))
415
416    raise SystemExit(2)
417
418def main():
419    from optparse import OptionParser
420
421    # Change stdout to be line-buffered.
422    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)
423
424    parser = OptionParser(usage = "%prog [options] -i INPUT -o OUTPUT",
425                          description =
426                          "Process featureset information")
427
428    parser.add_option("-i", "--in", dest = "fin", metavar = "<FD or FILE>",
429                      default = "0",
430                      help = "Featureset definitions")
431    parser.add_option("-o", "--out", dest = "fout", metavar = "<FD or FILE>",
432                      default = "1",
433                      help = "Featureset calculated information")
434
435    opts, _ = parser.parse_args()
436
437    if opts.fin is None or opts.fout is None:
438        parser.print_help(sys.stderr)
439        raise SystemExit(1)
440
441    state = State(opts.fin, opts.fout)
442
443    parse_definitions(state)
444    crunch_numbers(state)
445    write_results(state)
446
447
448if __name__ == "__main__":
449    try:
450        sys.exit(main())
451    except Fail, e:
452        print >>sys.stderr, "%s:" % (sys.argv[0],), e
453        sys.exit(1)
454    except SystemExit, e:
455        sys.exit(e.code)
456    except KeyboardInterrupt:
457        sys.exit(2)
458