1# vim: set syntax=python ts=4 :
2#
3# Copyright (c) 2018-2022 Intel Corporation
4# SPDX-License-Identifier: Apache-2.0
5
6import copy
7import warnings
8from typing import Any
9
10import scl
11from twisterlib.error import ConfigurationError
12
13
14def extract_fields_from_arg_list(
15    target_fields: set, arg_list: str | list
16 ) -> tuple[dict[str, list[str]], list[str]]:
17    """
18    Given a list of "FIELD=VALUE" args, extract values of args with a
19    given field name and return the remaining args separately.
20    """
21    extracted_fields: dict[str, list[str]] = {f: list() for f in target_fields}
22    other_fields: list[str] = []
23
24    if isinstance(arg_list, str):
25        args = arg_list.strip().split()
26    else:
27        args = arg_list
28
29    for field in args:
30        try:
31            name, val = field.split("=", 1)
32        except ValueError:
33            # Can't parse this. Just pass it through
34            other_fields.append(field)
35            continue
36
37        if name in target_fields:
38            extracted_fields[name].append(val.strip('\'"'))
39        else:
40            # Move to other_fields
41            other_fields.append(field)
42
43    return extracted_fields, other_fields
44
45
46class TwisterConfigParser:
47    """Class to read testsuite yaml files with semantic checking
48    """
49
50    testsuite_valid_keys: dict[str, dict[str, Any]] = {
51        "tags": {"type": "set", "required": False},
52        "type": {"type": "str", "default": "integration"},
53        "extra_args": {"type": "list"},
54        "extra_configs": {"type": "list"},
55        "extra_conf_files": {"type": "list", "default": []},
56        "extra_overlay_confs": {"type": "list", "default": []},
57        "extra_dtc_overlay_files": {"type": "list", "default": []},
58        "required_snippets": {"type": "list"},
59        "build_only": {"type": "bool", "default": False},
60        "build_on_all": {"type": "bool", "default": False},
61        "skip": {"type": "bool", "default": False},
62        "slow": {"type": "bool", "default": False},
63        "timeout": {"type": "int", "default": 60},
64        "min_ram": {"type": "int", "default": 16},
65        "modules": {"type": "list", "default": []},
66        "depends_on": {"type": "set"},
67        "min_flash": {"type": "int", "default": 32},
68        "arch_allow": {"type": "set"},
69        "arch_exclude": {"type": "set"},
70        "vendor_allow": {"type": "set"},
71        "vendor_exclude": {"type": "set"},
72        "extra_sections": {"type": "list", "default": []},
73        "integration_platforms": {"type": "list", "default": []},
74        "integration_toolchains": {"type": "list", "default": []},
75        "ignore_faults": {"type": "bool", "default": False},
76        "ignore_qemu_crash": {"type": "bool", "default": False},
77        "testcases": {"type": "list", "default": []},
78        "platform_type": {"type": "list", "default": []},
79        "platform_exclude": {"type": "set"},
80        "platform_allow": {"type": "set"},
81        "platform_key": {"type": "list", "default": []},
82        "simulation_exclude": {"type": "list", "default": []},
83        "toolchain_exclude": {"type": "set"},
84        "toolchain_allow": {"type": "set"},
85        "filter": {"type": "str"},
86        "levels": {"type": "list", "default": []},
87        "harness": {"type": "str", "default": "test"},
88        "harness_config": {"type": "map", "default": {}},
89        "seed": {"type": "int", "default": 0},
90        "sysbuild": {"type": "bool", "default": False}
91    }
92
93    def __init__(self, filename: str, schema: dict[str, Any]) -> None:
94        """Instantiate a new TwisterConfigParser object
95
96        @param filename Source .yaml file to read
97        """
98        self.schema = schema
99        self.filename = filename
100        self.data: dict[str, Any] = {}
101        self.scenarios: dict[str, Any] = {}
102        self.common: dict[str, Any] = {}
103
104    def load(self) -> dict[str, Any]:
105        data = scl.yaml_load_verify(self.filename, self.schema)
106        self.data = data
107
108        if 'tests' in self.data:
109            self.scenarios = self.data['tests']
110        if 'common' in self.data:
111            self.common = self.data['common']
112        return data
113
114    def _cast_value(self, value: Any, typestr: str) -> Any:
115        if typestr == "str":
116            return value.strip()
117
118        elif typestr == "float":
119            return float(value)
120
121        elif typestr == "int":
122            return int(value)
123
124        elif typestr == "bool":
125            return value
126
127        elif typestr.startswith("list"):
128            if isinstance(value, list):
129                return value
130            elif isinstance(value, str):
131                value = value.strip()
132                return [value] if value else list()
133            else:
134                raise ValueError
135
136        elif typestr.startswith("set"):
137            if isinstance(value, list):
138                return set(value)
139            elif isinstance(value, str):
140                value = value.strip()
141                return {value} if value else set()
142            else:
143                raise ValueError
144
145        elif typestr.startswith("map"):
146            return value
147        else:
148            raise ConfigurationError(self.filename, f"unknown type '{value}'")
149
150    def get_scenario(self, name: str) -> dict[str, Any]:
151        """Get a dictionary representing the keys/values within a scenario
152
153        @param name The scenario in the .yaml file to retrieve data from
154        @return A dictionary containing the scenario key-value pairs with
155            type conversion and default values filled in per valid_keys
156        """
157
158        # "CONF_FILE", "OVERLAY_CONFIG", and "DTC_OVERLAY_FILE" fields from each
159        # of the extra_args lines
160        extracted_common: dict = {}
161        extracted_testsuite: dict = {}
162
163        d: dict[str, Any] = {}
164        for k, v in self.common.items():
165            if k == "extra_args":
166                # Pull out these fields and leave the rest
167                extracted_common, d[k] = extract_fields_from_arg_list(
168                    {"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"}, v
169                )
170            else:
171                # Copy common value to avoid mutating it with test specific values below
172                d[k] = copy.copy(v)
173
174        for k, v in self.scenarios[name].items():
175            if k == "extra_args":
176                # Pull out these fields and leave the rest
177                extracted_testsuite, v = extract_fields_from_arg_list(
178                    {"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"}, v
179                )
180            if k in d:
181                if k == "filter":
182                    d[k] = f"({d[k]}) and ({v})"
183                elif k not in ("extra_conf_files", "extra_overlay_confs",
184                               "extra_dtc_overlay_files"):
185                    if isinstance(d[k], str) and isinstance(v, list):
186                        d[k] = [d[k]] + v
187                    elif isinstance(d[k], list) and isinstance(v, str):
188                        d[k] += [v]
189                    elif isinstance(d[k], list) and isinstance(v, list):
190                        d[k] += v
191                    elif isinstance(d[k], str) and isinstance(v, str):
192                        # overwrite if type is string, otherwise merge into a list
193                        type = self.testsuite_valid_keys[k]["type"]
194                        if type == "str":
195                            d[k] = v
196                        elif type in ("list", "set"):
197                            d[k] = [d[k], v]
198                        else:
199                            raise ValueError
200                    else:
201                        # replace value if not str/list (e.g. integer)
202                        d[k] = v
203            else:
204                d[k] = v
205
206        # Compile conf files in to a single list. The order to apply them is:
207        #  (1) CONF_FILEs extracted from common['extra_args']
208        #  (2) common['extra_conf_files']
209        #  (3) CONF_FILES extracted from scenarios[name]['extra_args']
210        #  (4) scenarios[name]['extra_conf_files']
211        d["extra_conf_files"] = \
212            extracted_common.get("CONF_FILE", []) + \
213            self.common.get("extra_conf_files", []) + \
214            extracted_testsuite.get("CONF_FILE", []) + \
215            self.scenarios[name].get("extra_conf_files", [])
216
217        # Repeat the above for overlay confs and DTC overlay files
218        d["extra_overlay_confs"] = \
219            extracted_common.get("OVERLAY_CONFIG", []) + \
220            self.common.get("extra_overlay_confs", []) + \
221            extracted_testsuite.get("OVERLAY_CONFIG", []) + \
222            self.scenarios[name].get("extra_overlay_confs", [])
223
224        d["extra_dtc_overlay_files"] = \
225            extracted_common.get("DTC_OVERLAY_FILE", []) + \
226            self.common.get("extra_dtc_overlay_files", []) + \
227            extracted_testsuite.get("DTC_OVERLAY_FILE", []) + \
228            self.scenarios[name].get("extra_dtc_overlay_files", [])
229
230        if any({len(x) > 0 for x in extracted_common.values()}) or \
231            any({len(x) > 0 for x in extracted_testsuite.values()}):
232            warnings.warn(
233                "Do not specify CONF_FILE, OVERLAY_CONFIG, or DTC_OVERLAY_FILE "
234                "in extra_args. This feature is deprecated and will soon "
235                "result in an error. Use extra_conf_files, extra_overlay_confs "
236                "or extra_dtc_overlay_files YAML fields instead",
237                DeprecationWarning,
238                stacklevel=2
239            )
240
241        for k, kinfo in self.testsuite_valid_keys.items():
242            if k not in d:
243                required = kinfo.get("required", False)
244
245                if required:
246                    raise ConfigurationError(
247                        self.filename,
248                        f"missing required value for '{k}' in test '{name}'"
249                    )
250                else:
251                    if "default" in kinfo:
252                        default = kinfo["default"]
253                    else:
254                        default = self._cast_value("", kinfo["type"])
255                    d[k] = default
256            else:
257                try:
258                    d[k] = self._cast_value(d[k], kinfo["type"])
259                except ValueError:
260                    raise ConfigurationError(
261                        self.filename,
262                        f"bad {kinfo['type']} value '{d[k]}' for key '{k}' in name '{name}'"
263                    ) from None
264
265        return d
266