1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * SN Platform GRU Driver
4  *
5  *            Dump GRU State
6  *
7  *  Copyright (c) 2008 Silicon Graphics, Inc.  All Rights Reserved.
8  */
9 
10 #include <linux/kernel.h>
11 #include <linux/mm.h>
12 #include <linux/spinlock.h>
13 #include <linux/uaccess.h>
14 #include <linux/delay.h>
15 #include <linux/bitops.h>
16 #include <asm/uv/uv_hub.h>
17 
18 #include <linux/nospec.h>
19 
20 #include "gru.h"
21 #include "grutables.h"
22 #include "gruhandles.h"
23 #include "grulib.h"
24 
25 #define CCH_LOCK_ATTEMPTS	10
26 
gru_user_copy_handle(void __user ** dp,void * s)27 static int gru_user_copy_handle(void __user **dp, void *s)
28 {
29 	if (copy_to_user(*dp, s, GRU_HANDLE_BYTES))
30 		return -1;
31 	*dp += GRU_HANDLE_BYTES;
32 	return 0;
33 }
34 
gru_dump_context_data(void * grubase,struct gru_context_configuration_handle * cch,void __user * ubuf,int ctxnum,int dsrcnt,int flush_cbrs)35 static int gru_dump_context_data(void *grubase,
36 			struct gru_context_configuration_handle *cch,
37 			void __user *ubuf, int ctxnum, int dsrcnt,
38 			int flush_cbrs)
39 {
40 	void *cb, *cbe, *tfh, *gseg;
41 	int i, scr;
42 
43 	gseg = grubase + ctxnum * GRU_GSEG_STRIDE;
44 	cb = gseg + GRU_CB_BASE;
45 	cbe = grubase + GRU_CBE_BASE;
46 	tfh = grubase + GRU_TFH_BASE;
47 
48 	for_each_cbr_in_allocation_map(i, &cch->cbr_allocation_map, scr) {
49 		if (flush_cbrs)
50 			gru_flush_cache(cb);
51 		if (gru_user_copy_handle(&ubuf, cb))
52 			goto fail;
53 		if (gru_user_copy_handle(&ubuf, tfh + i * GRU_HANDLE_STRIDE))
54 			goto fail;
55 		if (gru_user_copy_handle(&ubuf, cbe + i * GRU_HANDLE_STRIDE))
56 			goto fail;
57 		cb += GRU_HANDLE_STRIDE;
58 	}
59 	if (dsrcnt)
60 		memcpy(ubuf, gseg + GRU_DS_BASE, dsrcnt * GRU_HANDLE_STRIDE);
61 	return 0;
62 
63 fail:
64 	return -EFAULT;
65 }
66 
gru_dump_tfm(struct gru_state * gru,void __user * ubuf,void __user * ubufend)67 static int gru_dump_tfm(struct gru_state *gru,
68 		void __user *ubuf, void __user *ubufend)
69 {
70 	struct gru_tlb_fault_map *tfm;
71 	int i;
72 
73 	if (GRU_NUM_TFM * GRU_CACHE_LINE_BYTES > ubufend - ubuf)
74 		return -EFBIG;
75 
76 	for (i = 0; i < GRU_NUM_TFM; i++) {
77 		tfm = get_tfm(gru->gs_gru_base_vaddr, i);
78 		if (gru_user_copy_handle(&ubuf, tfm))
79 			goto fail;
80 	}
81 	return GRU_NUM_TFM * GRU_CACHE_LINE_BYTES;
82 
83 fail:
84 	return -EFAULT;
85 }
86 
gru_dump_tgh(struct gru_state * gru,void __user * ubuf,void __user * ubufend)87 static int gru_dump_tgh(struct gru_state *gru,
88 		void __user *ubuf, void __user *ubufend)
89 {
90 	struct gru_tlb_global_handle *tgh;
91 	int i;
92 
93 	if (GRU_NUM_TGH * GRU_CACHE_LINE_BYTES > ubufend - ubuf)
94 		return -EFBIG;
95 
96 	for (i = 0; i < GRU_NUM_TGH; i++) {
97 		tgh = get_tgh(gru->gs_gru_base_vaddr, i);
98 		if (gru_user_copy_handle(&ubuf, tgh))
99 			goto fail;
100 	}
101 	return GRU_NUM_TGH * GRU_CACHE_LINE_BYTES;
102 
103 fail:
104 	return -EFAULT;
105 }
106 
gru_dump_context(struct gru_state * gru,int ctxnum,void __user * ubuf,void __user * ubufend,char data_opt,char lock_cch,char flush_cbrs)107 static int gru_dump_context(struct gru_state *gru, int ctxnum,
108 		void __user *ubuf, void __user *ubufend, char data_opt,
109 		char lock_cch, char flush_cbrs)
110 {
111 	struct gru_dump_context_header hdr;
112 	struct gru_dump_context_header __user *uhdr = ubuf;
113 	struct gru_context_configuration_handle *cch, *ubufcch;
114 	struct gru_thread_state *gts;
115 	int try, cch_locked, cbrcnt = 0, dsrcnt = 0, bytes = 0, ret = 0;
116 	void *grubase;
117 
118 	memset(&hdr, 0, sizeof(hdr));
119 	grubase = gru->gs_gru_base_vaddr;
120 	cch = get_cch(grubase, ctxnum);
121 	for (try = 0; try < CCH_LOCK_ATTEMPTS; try++) {
122 		cch_locked =  trylock_cch_handle(cch);
123 		if (cch_locked)
124 			break;
125 		msleep(1);
126 	}
127 
128 	ubuf += sizeof(hdr);
129 	ubufcch = ubuf;
130 	if (gru_user_copy_handle(&ubuf, cch)) {
131 		if (cch_locked)
132 			unlock_cch_handle(cch);
133 		return -EFAULT;
134 	}
135 	if (cch_locked)
136 		ubufcch->delresp = 0;
137 	bytes = sizeof(hdr) + GRU_CACHE_LINE_BYTES;
138 
139 	if (cch_locked || !lock_cch) {
140 		gts = gru->gs_gts[ctxnum];
141 		if (gts && gts->ts_vma) {
142 			hdr.pid = gts->ts_tgid_owner;
143 			hdr.vaddr = gts->ts_vma->vm_start;
144 		}
145 		if (cch->state != CCHSTATE_INACTIVE) {
146 			cbrcnt = hweight64(cch->cbr_allocation_map) *
147 						GRU_CBR_AU_SIZE;
148 			dsrcnt = data_opt ? hweight32(cch->dsr_allocation_map) *
149 						GRU_DSR_AU_CL : 0;
150 		}
151 		bytes += (3 * cbrcnt + dsrcnt) * GRU_CACHE_LINE_BYTES;
152 		if (bytes > ubufend - ubuf)
153 			ret = -EFBIG;
154 		else
155 			ret = gru_dump_context_data(grubase, cch, ubuf, ctxnum,
156 							dsrcnt, flush_cbrs);
157 	}
158 	if (cch_locked)
159 		unlock_cch_handle(cch);
160 	if (ret)
161 		return ret;
162 
163 	hdr.magic = GRU_DUMP_MAGIC;
164 	hdr.gid = gru->gs_gid;
165 	hdr.ctxnum = ctxnum;
166 	hdr.cbrcnt = cbrcnt;
167 	hdr.dsrcnt = dsrcnt;
168 	hdr.cch_locked = cch_locked;
169 	if (copy_to_user(uhdr, &hdr, sizeof(hdr)))
170 		return -EFAULT;
171 
172 	return bytes;
173 }
174 
gru_dump_chiplet_request(unsigned long arg)175 int gru_dump_chiplet_request(unsigned long arg)
176 {
177 	struct gru_state *gru;
178 	struct gru_dump_chiplet_state_req req;
179 	void __user *ubuf;
180 	void __user *ubufend;
181 	int ctxnum, ret, cnt = 0;
182 
183 	if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
184 		return -EFAULT;
185 
186 	/* Currently, only dump by gid is implemented */
187 	if (req.gid >= gru_max_gids)
188 		return -EINVAL;
189 	req.gid = array_index_nospec(req.gid, gru_max_gids);
190 
191 	gru = GID_TO_GRU(req.gid);
192 	ubuf = req.buf;
193 	ubufend = req.buf + req.buflen;
194 
195 	ret = gru_dump_tfm(gru, ubuf, ubufend);
196 	if (ret < 0)
197 		goto fail;
198 	ubuf += ret;
199 
200 	ret = gru_dump_tgh(gru, ubuf, ubufend);
201 	if (ret < 0)
202 		goto fail;
203 	ubuf += ret;
204 
205 	for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
206 		if (req.ctxnum == ctxnum || req.ctxnum < 0) {
207 			ret = gru_dump_context(gru, ctxnum, ubuf, ubufend,
208 						req.data_opt, req.lock_cch,
209 						req.flush_cbrs);
210 			if (ret < 0)
211 				goto fail;
212 			ubuf += ret;
213 			cnt++;
214 		}
215 	}
216 
217 	if (copy_to_user((void __user *)arg, &req, sizeof(req)))
218 		return -EFAULT;
219 	return cnt;
220 
221 fail:
222 	return ret;
223 }
224