1 /*
2 * Copyright (c) 2011 NetApp, Inc.
3 * Copyright (c) 2018-2022 Intel Corporation.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29 #include <asm/guest/vm.h>
30 #include <asm/guest/ept.h>
31 #include <ptdev.h>
32 #include <vpci.h>
33 #include <asm/pci_dev.h>
34 #include <logmsg.h>
35 #include <delay.h>
36
37 #include "vpci_priv.h"
38
39 /**
40 * @pre pf_vdev != NULL
41 */
get_vf_devfun(const struct pci_vdev * pf_vdev,uint16_t fst_off,uint16_t stride,uint16_t id)42 static inline uint8_t get_vf_devfun(const struct pci_vdev *pf_vdev, uint16_t fst_off, uint16_t stride, uint16_t id)
43 {
44 return ((uint8_t)((pf_vdev->bdf.fields.devfun + fst_off + (stride * id)) & 0xFFU));
45 }
46
47 /**
48 * @pre pf_vdev != NULL
49 */
get_vf_bus(const struct pci_vdev * pf_vdev,uint16_t fst_off,uint16_t stride,uint16_t id)50 static inline uint8_t get_vf_bus(const struct pci_vdev *pf_vdev, uint16_t fst_off, uint16_t stride, uint16_t id)
51 {
52 return ((uint8_t)(pf_vdev->bdf.fields.bus + ((pf_vdev->bdf.fields.devfun + fst_off + (stride * id)) >> 8U)));
53 }
54
55 /**
56 * @pre pf_vdev != NULL
57 */
read_sriov_reg(const struct pci_vdev * pf_vdev,uint16_t reg)58 static inline uint16_t read_sriov_reg(const struct pci_vdev *pf_vdev, uint16_t reg)
59 {
60 return ((uint16_t)(pci_pdev_read_cfg(pf_vdev->bdf, pf_vdev->sriov.capoff + reg, 2U)));
61 }
62
63 /**
64 * @pre pf_vdev != NULL
65 */
is_vf_enabled(const struct pci_vdev * pf_vdev)66 static bool is_vf_enabled(const struct pci_vdev *pf_vdev)
67 {
68 uint16_t control;
69
70 control = read_sriov_reg(pf_vdev, PCIR_SRIOV_CONTROL);
71 return ((control & PCIM_SRIOV_VF_ENABLE) != 0U);
72 }
73
74 /**
75 * @pre pf_vdev != NULL
76 */
init_sriov_vf_bar(struct pci_vdev * pf_vdev)77 static void init_sriov_vf_bar(struct pci_vdev *pf_vdev)
78 {
79 init_vdev_pt(pf_vdev, true);
80 }
81
82 /**
83 * @pre pf_vdev != NULL
84 */
create_vf(struct pci_vdev * pf_vdev,union pci_bdf vf_bdf,uint16_t vf_id)85 static void create_vf(struct pci_vdev *pf_vdev, union pci_bdf vf_bdf, uint16_t vf_id)
86 {
87 struct pci_pdev *vf_pdev;
88 struct pci_vdev *vf_vdev = NULL;
89
90 /*
91 * Per VT-d 8.3.3, the VFs are under the scope of the same
92 * remapping unit as the associated PF when SRIOV is enabled.
93 */
94 vf_pdev = pci_init_pdev(vf_bdf, pf_vdev->pdev->drhd_index);
95 if (vf_pdev != NULL) {
96 struct acrn_vm_pci_dev_config *dev_cfg;
97
98 dev_cfg = init_one_dev_config(vf_pdev);
99 if (dev_cfg != NULL) {
100 vf_vdev = vpci_init_vdev(&vpci2vm(pf_vdev->vpci)->vpci, dev_cfg, pf_vdev);
101 }
102 }
103
104 /*
105 * if a VF vdev failed to be created, the VF number is less than requested number
106 * and the requested VF physical devices are ready at this time, clear VF_ENABLE.
107 */
108 if (vf_vdev == NULL) {
109 uint16_t control;
110
111 control = read_sriov_reg(pf_vdev, PCIR_SRIOV_CONTROL);
112 control &= (~PCIM_SRIOV_VF_ENABLE);
113 pci_pdev_write_cfg(pf_vdev->bdf, pf_vdev->sriov.capoff + PCIR_SRIOV_CONTROL, 2U, control);
114 pr_err("PF %x:%x.%x can't creat VF, unset VF_ENABLE",
115 pf_vdev->bdf.bits.b, pf_vdev->bdf.bits.d, pf_vdev->bdf.bits.f);
116 } else {
117 uint32_t bar_idx;
118 struct pci_vbar *vf_vbar;
119
120 /* VF bars information from its PF SRIOV capability, no need to access physical device */
121 vf_vdev->nr_bars = PCI_BAR_COUNT;
122 for (bar_idx = 0U; bar_idx < PCI_BAR_COUNT; bar_idx++) {
123 vf_vbar = &vf_vdev->vbars[bar_idx];
124 *vf_vbar = vf_vdev->phyfun->sriov.vbars[bar_idx];
125 vf_vbar->base_hpa += (vf_vbar->size * vf_id);
126 vf_vbar->base_gpa = vf_vbar->base_hpa;
127
128 /* Map VF's BARs when it's first created. */
129 if (vf_vbar->base_gpa != 0UL) {
130 struct acrn_vm *vm = vpci2vm(vf_vdev->vpci);
131 ept_add_mr(vm, (uint64_t *)(vm->arch_vm.nworld_eptp), vf_vbar->base_hpa,
132 vf_vbar->base_gpa, vf_vbar->size, EPT_WR | EPT_RD | EPT_UNCACHED);
133 }
134
135 if (has_msix_cap(vf_vdev) && (bar_idx == vf_vdev->msix.table_bar)) {
136 vf_vdev->msix.mmio_hpa = vf_vbar->base_hpa;
137 vf_vdev->msix.mmio_size = vf_vbar->size;
138 vdev_pt_map_msix(vf_vdev, false);
139 }
140
141 /*
142 * VF BARs value are zero and read only, according to PCI Express
143 * Base 4.0 chapter 9.3.4.1.11, the VF
144 */
145 pci_vdev_write_vcfg(vf_vdev, pci_bar_offset(bar_idx), 4U, 0U);
146 }
147 }
148 }
149
150 /**
151 * @pre pf_vdev != NULL
152 * @pre is_vf_enabled(pf_dev) == true
153 * @Application constraints: PCIR_SRIOV_NUMVFS register value cannot be 0 if VF_ENABLE is set.
154 */
enable_vfs(struct pci_vdev * pf_vdev)155 static void enable_vfs(struct pci_vdev *pf_vdev)
156 {
157 union pci_bdf vf_bdf;
158 uint16_t idx;
159 uint16_t sub_vid = 0U;
160 uint16_t num_vfs, stride, fst_off;
161
162 /* Confirm that the physical VF_ENABLE register has been set successfully */
163 ASSERT(is_vf_enabled(pf_vdev), "VF_ENABLE was not set successfully on the hardware");
164
165 /*
166 * All VFs bars information are located at PF VF_BAR fields of SRIOV capability.
167 * Initialize the PF's VF_BAR registers before initialize each VF device bar.
168 */
169 init_sriov_vf_bar(pf_vdev);
170
171 /*
172 * Per PCIE base spec 9.3.3.3.1, VF Enable bit from cleared to set, the
173 * system is not perrmitted to issue requests to the VFs until one of
174 * the following is true:
175 * 1. at least 100ms has passed.
176 * 2. An FRS message has been received from the PF with a reason code
177 * of VF Enabled.
178 * 3. At least VF Enable Time has passed since VF Enable was Set.
179 * VF Enable Time is either the Reset Time value in the Readiness Time
180 * Reporting capability associated with the VF or a value determined
181 * by system software/firmware.
182 *
183 * Curerntly, we use the first way to wait for VF physical devices to be ready.
184 */
185 udelay (100U * 1000U);
186
187 /*
188 * Due to VF's DEVICE ID and VENDOR ID are 0xFFFF, so check if VF physical
189 * device has been created by the value of SUBSYSTEM VENDOR ID.
190 * To check if all enabled VFs are ready, just check the first VF already exists,
191 * do not need to check all.
192 */
193 fst_off = read_sriov_reg(pf_vdev, PCIR_SRIOV_FST_VF_OFF);
194 stride = read_sriov_reg(pf_vdev, PCIR_SRIOV_VF_STRIDE);
195 vf_bdf.fields.bus = get_vf_bus(pf_vdev, fst_off, stride, 0U);
196 vf_bdf.fields.devfun = get_vf_devfun(pf_vdev, fst_off, stride, 0U);
197 sub_vid = (uint16_t) pci_pdev_read_cfg(vf_bdf, PCIV_SUB_VENDOR_ID, 2U);
198 if ((sub_vid != 0xFFFFU) && (sub_vid != 0U)) {
199 struct pci_vdev *vf_vdev;
200
201 num_vfs = read_sriov_reg(pf_vdev, PCIR_SRIOV_NUMVFS);
202 for (idx = 0U; idx < num_vfs; idx++) {
203 vf_bdf.fields.bus = get_vf_bus(pf_vdev, fst_off, stride, idx);
204 vf_bdf.fields.devfun = get_vf_devfun(pf_vdev, fst_off, stride, idx);
205
206 /*
207 * If one VF has never been created then create new one including pdev/vdev structures.
208 *
209 * The VF maybe have already existed but it is a zombie instance that vf_vdev->vpci
210 * is NULL, in this case, we need to make the vf_vdev available again in here.
211 */
212 vf_vdev = pci_find_vdev(&vpci2vm(pf_vdev->vpci)->vpci, vf_bdf);
213 if (vf_vdev == NULL) {
214 create_vf(pf_vdev, vf_bdf, idx);
215 } else {
216 /* Re-activate a zombie VF */
217 if (is_zombie_vf(vf_vdev)) {
218 vf_vdev->vdev_ops->init_vdev(vf_vdev);
219 }
220 }
221 }
222 } else {
223 /*
224 * If the VF physical device was not created successfully, the pdev/vdev
225 * will also not be created so that Service VM can aware of VF creation failure,
226 */
227 pr_err("PF %x:%x.%x can't create VFs after 100 ms",
228 pf_vdev->bdf.bits.b, pf_vdev->bdf.bits.d, pf_vdev->bdf.bits.f);
229 }
230 }
231
232 /**
233 * @pre pf_vdev != NULL
234 */
disable_vfs(struct pci_vdev * pf_vdev)235 static void disable_vfs(struct pci_vdev *pf_vdev)
236 {
237 uint16_t idx, num_vfs, stride, first;
238 struct pci_vdev *vf_vdev;
239
240 /*
241 * PF can disable VFs only when all VFs are not used by any VM or any application
242 *
243 * Ideally, VF instances should be deleted after VFs are disabled, but for FuSa reasons,
244 * we simply set the VF instance status to "zombie" to avoid dynamically adding/removing
245 * resources
246 *
247 * If the VF drivers are still running in Service VM or User VM, the MMIO access will return 0xFF.
248 */
249 num_vfs = read_sriov_reg(pf_vdev, PCIR_SRIOV_NUMVFS);
250 first = read_sriov_reg(pf_vdev, PCIR_SRIOV_FST_VF_OFF);
251 stride = read_sriov_reg(pf_vdev, PCIR_SRIOV_VF_STRIDE);
252 for (idx = 0U; idx < num_vfs; idx++) {
253 union pci_bdf bdf;
254
255 bdf.fields.bus = get_vf_bus(pf_vdev, first, stride, idx);
256 bdf.fields.devfun = get_vf_devfun(pf_vdev, first, stride, idx);
257 vf_vdev = pci_find_vdev(&vpci2vm(pf_vdev->vpci)->vpci, bdf);
258 if ((vf_vdev != NULL) && (!is_zombie_vf(vf_vdev))) {
259 /* set disabled VF as zombie vdev instance */
260 vf_vdev->vdev_ops->deinit_vdev(vf_vdev);
261 }
262 }
263 }
264
265 /**
266 * @pre vdev != NULL
267 * @pre vdev->pdev != NULL
268 */
init_vsriov(struct pci_vdev * vdev)269 void init_vsriov(struct pci_vdev *vdev)
270 {
271 struct pci_pdev *pdev = vdev->pdev;
272
273 vdev->sriov.capoff = pdev->sriov.capoff;
274 vdev->sriov.caplen = pdev->sriov.caplen;
275 }
276
277 /**
278 * @pre vdev != NULL
279 * @pre vdev->pdev != NULL
280 */
read_sriov_cap_reg(const struct pci_vdev * vdev,uint32_t offset,uint32_t bytes,uint32_t * val)281 void read_sriov_cap_reg(const struct pci_vdev *vdev, uint32_t offset, uint32_t bytes, uint32_t *val)
282 {
283 if (!vdev->pdev->sriov.hide_sriov) {
284 /* no need to do emulation, passthrough to physical device directly */
285 *val = pci_pdev_read_cfg(vdev->pdev->bdf, offset, bytes);
286 } else {
287 *val = 0xffffffffU;
288 }
289 }
290
291 /**
292 * @pre vdev != NULL
293 * @pre vdev->pdev != NULL
294 */
write_sriov_cap_reg(struct pci_vdev * vdev,uint32_t offset,uint32_t bytes,uint32_t val)295 void write_sriov_cap_reg(struct pci_vdev *vdev, uint32_t offset, uint32_t bytes, uint32_t val)
296 {
297
298 uint32_t reg;
299
300 reg = offset - vdev->sriov.capoff;
301 if (!vdev->pdev->sriov.hide_sriov) {
302 if (reg == PCIR_SRIOV_CONTROL) {
303 bool enable;
304
305 enable = ((val & PCIM_SRIOV_VF_ENABLE) != 0U);
306 if (enable != is_vf_enabled(vdev)) {
307 if (enable) {
308 /*
309 * set VF_ENABLE to PF physical device before enable_vfs
310 * since need to ask hardware to create VF physical
311 * devices firstly
312 */
313 pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
314 enable_vfs(vdev);
315 } else {
316 disable_vfs(vdev);
317 pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
318 }
319 } else {
320 pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
321 }
322 } else if (reg == PCIR_SRIOV_NUMVFS) {
323 uint16_t total;
324
325 total = read_sriov_reg(vdev, PCIR_SRIOV_TOTAL_VFS);
326 /*
327 * sanity check for NumVFs register based on PCE Express Base 4.0 9.3.3.7 chapter
328 * The results are undefined if NumVFs is set to a value greater than TotalVFs
329 * NumVFs may only be written while VF Enable is Clear
330 * If NumVFs is written when VF Enable is Set, the results are undefined
331 */
332 if ((((uint16_t)(val & 0xFFU)) <= total) && (!is_vf_enabled(vdev))) {
333 pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
334 }
335 } else {
336 pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
337 }
338 }
339 }
340
341
342 /**
343 * @pre vdev != NULL
344 */
sriov_bar_offset(const struct pci_vdev * vdev,uint32_t bar_idx)345 uint32_t sriov_bar_offset(const struct pci_vdev *vdev, uint32_t bar_idx)
346 {
347 return (vdev->sriov.capoff + PCIR_SRIOV_VF_BAR_OFF + (bar_idx << 2U));
348 }
349