1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3 
4 #include <linux/interrupt.h>
5 #include <linux/iopoll.h>
6 #include <linux/device.h>
7 
8 #include "lima_device.h"
9 #include "lima_mmu.h"
10 #include "lima_vm.h"
11 #include "lima_regs.h"
12 
13 #define mmu_write(reg, data) writel(data, ip->iomem + reg)
14 #define mmu_read(reg) readl(ip->iomem + reg)
15 
16 #define lima_mmu_send_command(cmd, addr, val, cond)	     \
17 ({							     \
18 	int __ret;					     \
19 							     \
20 	mmu_write(LIMA_MMU_COMMAND, cmd);		     \
21 	__ret = readl_poll_timeout(ip->iomem + (addr), val,  \
22 				  cond, 0, 100);	     \
23 	if (__ret)					     \
24 		dev_err(dev->dev,			     \
25 			"mmu command %x timeout\n", cmd);    \
26 	__ret;						     \
27 })
28 
lima_mmu_irq_handler(int irq,void * data)29 static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
30 {
31 	struct lima_ip *ip = data;
32 	struct lima_device *dev = ip->dev;
33 	u32 status = mmu_read(LIMA_MMU_INT_STATUS);
34 	struct lima_sched_pipe *pipe;
35 
36 	/* for shared irq case */
37 	if (!status)
38 		return IRQ_NONE;
39 
40 	if (status & LIMA_MMU_INT_PAGE_FAULT) {
41 		u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR);
42 
43 		dev_err(dev->dev, "mmu page fault at 0x%x from bus id %d of type %s on %s\n",
44 			fault, LIMA_MMU_STATUS_BUS_ID(status),
45 			status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read",
46 			lima_ip_name(ip));
47 	}
48 
49 	if (status & LIMA_MMU_INT_READ_BUS_ERROR)
50 		dev_err(dev->dev, "mmu %s irq bus error\n", lima_ip_name(ip));
51 
52 	/* mask all interrupts before resume */
53 	mmu_write(LIMA_MMU_INT_MASK, 0);
54 	mmu_write(LIMA_MMU_INT_CLEAR, status);
55 
56 	pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp);
57 	lima_sched_pipe_mmu_error(pipe);
58 
59 	return IRQ_HANDLED;
60 }
61 
lima_mmu_hw_init(struct lima_ip * ip)62 static int lima_mmu_hw_init(struct lima_ip *ip)
63 {
64 	struct lima_device *dev = ip->dev;
65 	int err;
66 	u32 v;
67 
68 	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
69 	err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
70 				    LIMA_MMU_DTE_ADDR, v, v == 0);
71 	if (err)
72 		return err;
73 
74 	mmu_write(LIMA_MMU_INT_MASK,
75 		  LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
76 	mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
77 	return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
78 				     LIMA_MMU_STATUS, v,
79 				     v & LIMA_MMU_STATUS_PAGING_ENABLED);
80 }
81 
lima_mmu_resume(struct lima_ip * ip)82 int lima_mmu_resume(struct lima_ip *ip)
83 {
84 	if (ip->id == lima_ip_ppmmu_bcast)
85 		return 0;
86 
87 	return lima_mmu_hw_init(ip);
88 }
89 
lima_mmu_suspend(struct lima_ip * ip)90 void lima_mmu_suspend(struct lima_ip *ip)
91 {
92 
93 }
94 
lima_mmu_init(struct lima_ip * ip)95 int lima_mmu_init(struct lima_ip *ip)
96 {
97 	struct lima_device *dev = ip->dev;
98 	int err;
99 
100 	if (ip->id == lima_ip_ppmmu_bcast)
101 		return 0;
102 
103 	mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
104 	if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) {
105 		dev_err(dev->dev, "mmu %s dte write test fail\n", lima_ip_name(ip));
106 		return -EIO;
107 	}
108 
109 	err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
110 			       IRQF_SHARED, lima_ip_name(ip), ip);
111 	if (err) {
112 		dev_err(dev->dev, "mmu %s fail to request irq\n", lima_ip_name(ip));
113 		return err;
114 	}
115 
116 	return lima_mmu_hw_init(ip);
117 }
118 
lima_mmu_fini(struct lima_ip * ip)119 void lima_mmu_fini(struct lima_ip *ip)
120 {
121 
122 }
123 
lima_mmu_flush_tlb(struct lima_ip * ip)124 void lima_mmu_flush_tlb(struct lima_ip *ip)
125 {
126 	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
127 }
128 
lima_mmu_switch_vm(struct lima_ip * ip,struct lima_vm * vm)129 void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
130 {
131 	struct lima_device *dev = ip->dev;
132 	u32 v;
133 
134 	lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL,
135 			      LIMA_MMU_STATUS, v,
136 			      v & LIMA_MMU_STATUS_STALL_ACTIVE);
137 
138 	mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
139 
140 	/* flush the TLB */
141 	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
142 
143 	lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL,
144 			      LIMA_MMU_STATUS, v,
145 			      !(v & LIMA_MMU_STATUS_STALL_ACTIVE));
146 }
147 
lima_mmu_page_fault_resume(struct lima_ip * ip)148 void lima_mmu_page_fault_resume(struct lima_ip *ip)
149 {
150 	struct lima_device *dev = ip->dev;
151 	u32 status = mmu_read(LIMA_MMU_STATUS);
152 	u32 v;
153 
154 	if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) {
155 		dev_info(dev->dev, "mmu resume\n");
156 
157 		mmu_write(LIMA_MMU_INT_MASK, 0);
158 		mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
159 		lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
160 				      LIMA_MMU_DTE_ADDR, v, v == 0);
161 		mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
162 		mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
163 		lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
164 				      LIMA_MMU_STATUS, v,
165 				      v & LIMA_MMU_STATUS_PAGING_ENABLED);
166 	}
167 }
168