1 /*
2  * Copyright (C) 2011 Advanced Micro Devices, Inc.
3  * Author: Leo Duran <leo.duran@amd.com>
4  * Author: Wei Wang <wei.wang2@amd.com> - adapted to xen
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "iommu.h"
21 #include "../ats.h"
22 
23 #define CMD_COMPLETION_INIT 0
24 #define CMD_COMPLETION_DONE 1
25 
send_iommu_command(struct amd_iommu * iommu,const uint32_t cmd[4])26 static void send_iommu_command(struct amd_iommu *iommu,
27                                const uint32_t cmd[4])
28 {
29     uint32_t tail;
30     unsigned long flags;
31 
32     spin_lock_irqsave(&iommu->lock, flags);
33 
34     tail = iommu->cmd_buffer.tail + sizeof(cmd_entry_t);
35     if ( tail == iommu->cmd_buffer.size )
36         tail = 0;
37 
38     while ( tail == (readl(iommu->mmio_base +
39                            IOMMU_CMD_BUFFER_HEAD_OFFSET) &
40                      IOMMU_RING_BUFFER_PTR_MASK) )
41     {
42         printk_once(XENLOG_ERR "AMD IOMMU %pp: no cmd slot available\n",
43                     &PCI_SBDF(iommu->seg, iommu->bdf));
44         cpu_relax();
45     }
46 
47     memcpy(iommu->cmd_buffer.buffer + iommu->cmd_buffer.tail,
48            cmd, sizeof(cmd_entry_t));
49 
50     iommu->cmd_buffer.tail = tail;
51 
52     writel(tail, iommu->mmio_base + IOMMU_CMD_BUFFER_TAIL_OFFSET);
53 
54     spin_unlock_irqrestore(&iommu->lock, flags);
55 }
56 
flush_command_buffer(struct amd_iommu * iommu,unsigned int timeout_base)57 static void flush_command_buffer(struct amd_iommu *iommu,
58                                  unsigned int timeout_base)
59 {
60     static DEFINE_PER_CPU(uint64_t, poll_slot);
61     uint64_t *this_poll_slot = &this_cpu(poll_slot);
62     paddr_t addr = virt_to_maddr(this_poll_slot);
63     /* send a COMPLETION_WAIT command to flush command buffer */
64     uint32_t cmd[4] = {
65         addr | MASK_INSR(IOMMU_CONTROL_ENABLED,
66                          IOMMU_COMP_WAIT_S_FLAG_MASK),
67         (addr >> 32) | MASK_INSR(IOMMU_CMD_COMPLETION_WAIT,
68                                  IOMMU_CMD_OPCODE_MASK),
69         CMD_COMPLETION_DONE,
70         0
71     };
72     s_time_t start, timeout;
73     static unsigned int __read_mostly threshold = 1;
74 
75     ACCESS_ONCE(*this_poll_slot) = CMD_COMPLETION_INIT;
76 
77     send_iommu_command(iommu, cmd);
78 
79     start = NOW();
80     timeout = start + (timeout_base ?: 100) * MILLISECS(threshold);
81     while ( ACCESS_ONCE(*this_poll_slot) != CMD_COMPLETION_DONE )
82     {
83         if ( timeout && NOW() > timeout )
84         {
85             threshold |= threshold << 1;
86             printk(XENLOG_WARNING
87                    "AMD IOMMU %pp: %scompletion wait taking too long\n",
88                    &PCI_SBDF(iommu->seg, iommu->bdf),
89                    timeout_base ? "iotlb " : "");
90             timeout = 0;
91         }
92         cpu_relax();
93     }
94 
95     if ( !timeout )
96         printk(XENLOG_WARNING
97                "AMD IOMMU %pp: %scompletion wait took %lums\n",
98                &PCI_SBDF(iommu->seg, iommu->bdf),
99                timeout_base ? "iotlb " : "",
100                (NOW() - start) / 10000000);
101 }
102 
103 /* Build low level iommu command messages */
invalidate_iommu_pages(struct amd_iommu * iommu,u64 io_addr,u16 domain_id,u16 order)104 static void invalidate_iommu_pages(struct amd_iommu *iommu,
105                                    u64 io_addr, u16 domain_id, u16 order)
106 {
107     u64 addr_lo, addr_hi;
108     u32 cmd[4], entry;
109     int sflag = 0, pde = 0;
110 
111     ASSERT ( order == 0 || order == 9 || order == 18 );
112 
113     /* All pages associated with the domainID are invalidated */
114     if ( order || (io_addr == INV_IOMMU_ALL_PAGES_ADDRESS ) )
115     {
116         sflag = 1;
117         pde = 1;
118     }
119 
120     /* If sflag == 1, the size of the invalidate command is determined
121      by the first zero bit in the address starting from Address[12] */
122     if ( order )
123     {
124         u64 mask = 1ULL << (order - 1 + PAGE_SHIFT);
125         io_addr &= ~mask;
126         io_addr |= mask - 1;
127     }
128 
129     addr_lo = io_addr & DMA_32BIT_MASK;
130     addr_hi = io_addr >> 32;
131 
132     set_field_in_reg_u32(domain_id, 0,
133                          IOMMU_INV_IOMMU_PAGES_DOMAIN_ID_MASK,
134                          IOMMU_INV_IOMMU_PAGES_DOMAIN_ID_SHIFT, &entry);
135     set_field_in_reg_u32(IOMMU_CMD_INVALIDATE_IOMMU_PAGES, entry,
136                          IOMMU_CMD_OPCODE_MASK, IOMMU_CMD_OPCODE_SHIFT,
137                          &entry);
138     cmd[1] = entry;
139 
140     set_field_in_reg_u32(sflag, 0,
141                          IOMMU_INV_IOMMU_PAGES_S_FLAG_MASK,
142                          IOMMU_INV_IOMMU_PAGES_S_FLAG_SHIFT, &entry);
143     set_field_in_reg_u32(pde, entry,
144                          IOMMU_INV_IOMMU_PAGES_PDE_FLAG_MASK,
145                          IOMMU_INV_IOMMU_PAGES_PDE_FLAG_SHIFT, &entry);
146     set_field_in_reg_u32((u32)addr_lo >> PAGE_SHIFT, entry,
147                          IOMMU_INV_IOMMU_PAGES_ADDR_LOW_MASK,
148                          IOMMU_INV_IOMMU_PAGES_ADDR_LOW_SHIFT, &entry);
149     cmd[2] = entry;
150 
151     set_field_in_reg_u32((u32)addr_hi, 0,
152                          IOMMU_INV_IOMMU_PAGES_ADDR_HIGH_MASK,
153                          IOMMU_INV_IOMMU_PAGES_ADDR_HIGH_SHIFT, &entry);
154     cmd[3] = entry;
155 
156     cmd[0] = 0;
157     send_iommu_command(iommu, cmd);
158 }
159 
invalidate_iotlb_pages(struct amd_iommu * iommu,u16 maxpend,u32 pasid,u16 queueid,u64 io_addr,u16 dev_id,u16 order)160 static void invalidate_iotlb_pages(struct amd_iommu *iommu,
161                                    u16 maxpend, u32 pasid, u16 queueid,
162                                    u64 io_addr, u16 dev_id, u16 order)
163 {
164     u64 addr_lo, addr_hi;
165     u32 cmd[4], entry;
166     int sflag = 0;
167 
168     ASSERT ( order == 0 || order == 9 || order == 18 );
169 
170     if ( order || (io_addr == INV_IOMMU_ALL_PAGES_ADDRESS ) )
171         sflag = 1;
172 
173     /* If sflag == 1, the size of the invalidate command is determined
174      by the first zero bit in the address starting from Address[12] */
175     if ( order )
176     {
177         u64 mask = 1ULL << (order - 1 + PAGE_SHIFT);
178         io_addr &= ~mask;
179         io_addr |= mask - 1;
180     }
181 
182     addr_lo = io_addr & DMA_32BIT_MASK;
183     addr_hi = io_addr >> 32;
184 
185     set_field_in_reg_u32(dev_id, 0,
186                          IOMMU_INV_IOTLB_PAGES_DEVICE_ID_MASK,
187                          IOMMU_INV_IOTLB_PAGES_DEVICE_ID_SHIFT, &entry);
188 
189     set_field_in_reg_u32(maxpend, entry,
190                          IOMMU_INV_IOTLB_PAGES_MAXPEND_MASK,
191                          IOMMU_INV_IOTLB_PAGES_MAXPEND_SHIFT, &entry);
192 
193     set_field_in_reg_u32(pasid & 0xff, entry,
194                          IOMMU_INV_IOTLB_PAGES_PASID1_MASK,
195                          IOMMU_INV_IOTLB_PAGES_PASID1_SHIFT, &entry);
196     cmd[0] = entry;
197 
198     set_field_in_reg_u32(IOMMU_CMD_INVALIDATE_IOTLB_PAGES, 0,
199                          IOMMU_CMD_OPCODE_MASK, IOMMU_CMD_OPCODE_SHIFT,
200                          &entry);
201 
202     set_field_in_reg_u32(pasid >> 8, entry,
203                          IOMMU_INV_IOTLB_PAGES_PASID2_MASK,
204                          IOMMU_INV_IOTLB_PAGES_PASID2_SHIFT,
205                          &entry);
206 
207     set_field_in_reg_u32(queueid, entry,
208                          IOMMU_INV_IOTLB_PAGES_QUEUEID_MASK,
209                          IOMMU_INV_IOTLB_PAGES_QUEUEID_SHIFT,
210                          &entry);
211     cmd[1] = entry;
212 
213     set_field_in_reg_u32(sflag, 0,
214                          IOMMU_INV_IOTLB_PAGES_S_FLAG_MASK,
215                          IOMMU_INV_IOTLB_PAGES_S_FLAG_MASK, &entry);
216 
217     set_field_in_reg_u32((u32)addr_lo >> PAGE_SHIFT, entry,
218                          IOMMU_INV_IOTLB_PAGES_ADDR_LOW_MASK,
219                          IOMMU_INV_IOTLB_PAGES_ADDR_LOW_SHIFT, &entry);
220     cmd[2] = entry;
221 
222     set_field_in_reg_u32((u32)addr_hi, 0,
223                          IOMMU_INV_IOTLB_PAGES_ADDR_HIGH_MASK,
224                          IOMMU_INV_IOTLB_PAGES_ADDR_HIGH_SHIFT, &entry);
225     cmd[3] = entry;
226 
227     send_iommu_command(iommu, cmd);
228 }
229 
invalidate_dev_table_entry(struct amd_iommu * iommu,u16 device_id)230 static void invalidate_dev_table_entry(struct amd_iommu *iommu,
231                                        u16 device_id)
232 {
233     u32 cmd[4], entry;
234 
235     cmd[3] = cmd[2] = 0;
236     set_field_in_reg_u32(device_id, 0,
237                          IOMMU_INV_DEVTAB_ENTRY_DEVICE_ID_MASK,
238                          IOMMU_INV_DEVTAB_ENTRY_DEVICE_ID_SHIFT, &entry);
239     cmd[0] = entry;
240 
241     set_field_in_reg_u32(IOMMU_CMD_INVALIDATE_DEVTAB_ENTRY, 0,
242                          IOMMU_CMD_OPCODE_MASK, IOMMU_CMD_OPCODE_SHIFT,
243                          &entry);
244     cmd[1] = entry;
245 
246     send_iommu_command(iommu, cmd);
247 }
248 
invalidate_interrupt_table(struct amd_iommu * iommu,u16 device_id)249 static void invalidate_interrupt_table(struct amd_iommu *iommu, u16 device_id)
250 {
251     u32 cmd[4], entry;
252 
253     cmd[3] = cmd[2] = 0;
254     set_field_in_reg_u32(device_id, 0,
255                          IOMMU_INV_INT_TABLE_DEVICE_ID_MASK,
256                          IOMMU_INV_INT_TABLE_DEVICE_ID_SHIFT, &entry);
257     cmd[0] = entry;
258     set_field_in_reg_u32(IOMMU_CMD_INVALIDATE_INT_TABLE, 0,
259                          IOMMU_CMD_OPCODE_MASK, IOMMU_CMD_OPCODE_SHIFT,
260                          &entry);
261     cmd[1] = entry;
262     send_iommu_command(iommu, cmd);
263 }
264 
invalidate_iommu_all(struct amd_iommu * iommu)265 static void invalidate_iommu_all(struct amd_iommu *iommu)
266 {
267     u32 cmd[4], entry;
268 
269     cmd[3] = cmd[2] = cmd[0] = 0;
270 
271     set_field_in_reg_u32(IOMMU_CMD_INVALIDATE_IOMMU_ALL, 0,
272                          IOMMU_CMD_OPCODE_MASK, IOMMU_CMD_OPCODE_SHIFT,
273                          &entry);
274     cmd[1] = entry;
275 
276     send_iommu_command(iommu, cmd);
277 }
278 
amd_iommu_flush_iotlb(u8 devfn,const struct pci_dev * pdev,daddr_t daddr,unsigned int order)279 void amd_iommu_flush_iotlb(u8 devfn, const struct pci_dev *pdev,
280                            daddr_t daddr, unsigned int order)
281 {
282     struct amd_iommu *iommu;
283     unsigned int req_id, queueid, maxpend;
284 
285     if ( !ats_enabled )
286         return;
287 
288     if ( !pci_ats_enabled(pdev->seg, pdev->bus, pdev->devfn) )
289         return;
290 
291     iommu = find_iommu_for_device(pdev->seg, pdev->sbdf.bdf);
292 
293     if ( !iommu )
294     {
295         AMD_IOMMU_WARN("can't find IOMMU for %pp\n",
296                        &PCI_SBDF(pdev->seg, pdev->bus, devfn));
297         return;
298     }
299 
300     if ( !iommu_has_cap(iommu, PCI_CAP_IOTLB_SHIFT) )
301         return;
302 
303     req_id = get_dma_requestor_id(iommu->seg, PCI_BDF(pdev->bus, devfn));
304     queueid = req_id;
305     maxpend = pdev->ats.queue_depth & 0xff;
306 
307     /* send INVALIDATE_IOTLB_PAGES command */
308     invalidate_iotlb_pages(iommu, maxpend, 0, queueid, daddr, req_id, order);
309     flush_command_buffer(iommu, iommu_dev_iotlb_timeout);
310 }
311 
amd_iommu_flush_all_iotlbs(const struct domain * d,daddr_t daddr,unsigned int order)312 static void amd_iommu_flush_all_iotlbs(const struct domain *d, daddr_t daddr,
313                                        unsigned int order)
314 {
315     struct pci_dev *pdev;
316 
317     for_each_pdev( d, pdev )
318     {
319         u8 devfn = pdev->devfn;
320 
321         do {
322             amd_iommu_flush_iotlb(devfn, pdev, daddr, order);
323             devfn += pdev->phantom_stride;
324         } while ( devfn != pdev->devfn &&
325                   PCI_SLOT(devfn) == PCI_SLOT(pdev->devfn) );
326     }
327 }
328 
329 /* Flush iommu cache after p2m changes. */
_amd_iommu_flush_pages(struct domain * d,daddr_t daddr,unsigned int order)330 static void _amd_iommu_flush_pages(struct domain *d,
331                                    daddr_t daddr, unsigned int order)
332 {
333     struct amd_iommu *iommu;
334     unsigned int dom_id = d->domain_id;
335 
336     /* send INVALIDATE_IOMMU_PAGES command */
337     for_each_amd_iommu ( iommu )
338     {
339         invalidate_iommu_pages(iommu, daddr, dom_id, order);
340         flush_command_buffer(iommu, 0);
341     }
342 
343     if ( ats_enabled )
344     {
345         amd_iommu_flush_all_iotlbs(d, daddr, order);
346 
347         /*
348          * Hidden devices are associated with DomXEN but usable by the
349          * hardware domain. Hence they need dealing with here as well.
350          */
351         if ( is_hardware_domain(d) )
352             amd_iommu_flush_all_iotlbs(dom_xen, daddr, order);
353     }
354 }
355 
amd_iommu_flush_all_pages(struct domain * d)356 void amd_iommu_flush_all_pages(struct domain *d)
357 {
358     _amd_iommu_flush_pages(d, INV_IOMMU_ALL_PAGES_ADDRESS, 0);
359 }
360 
amd_iommu_flush_pages(struct domain * d,unsigned long dfn,unsigned int order)361 void amd_iommu_flush_pages(struct domain *d,
362                            unsigned long dfn, unsigned int order)
363 {
364     _amd_iommu_flush_pages(d, __dfn_to_daddr(dfn), order);
365 }
366 
amd_iommu_flush_device(struct amd_iommu * iommu,uint16_t bdf,domid_t domid)367 void amd_iommu_flush_device(struct amd_iommu *iommu, uint16_t bdf,
368                             domid_t domid)
369 {
370     invalidate_dev_table_entry(iommu, bdf);
371     flush_command_buffer(iommu, 0);
372 
373     /* Also invalidate IOMMU TLB entries when flushing the DTE. */
374     if ( domid != DOMID_INVALID )
375     {
376         invalidate_iommu_pages(iommu, INV_IOMMU_ALL_PAGES_ADDRESS, domid, 0);
377         flush_command_buffer(iommu, 0);
378     }
379 }
380 
amd_iommu_flush_intremap(struct amd_iommu * iommu,uint16_t bdf)381 void amd_iommu_flush_intremap(struct amd_iommu *iommu, uint16_t bdf)
382 {
383     invalidate_interrupt_table(iommu, bdf);
384     flush_command_buffer(iommu, 0);
385 }
386 
amd_iommu_flush_all_caches(struct amd_iommu * iommu)387 void amd_iommu_flush_all_caches(struct amd_iommu *iommu)
388 {
389     invalidate_iommu_all(iommu);
390     flush_command_buffer(iommu, 0);
391 }
392