1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/types.h>
4 #include <linux/mutex.h>
5 #include <linux/slab.h>
6 #include <linux/iopoll.h>
7 #include <linux/pci.h>
8 #include <linux/dma-mapping.h>
9 #include "adf_accel_devices.h"
10 #include "adf_admin.h"
11 #include "adf_common_drv.h"
12 #include "adf_cfg.h"
13 #include "adf_heartbeat.h"
14 #include "icp_qat_fw_init_admin.h"
15
16 #define ADF_ADMIN_MAILBOX_STRIDE 0x1000
17 #define ADF_ADMINMSG_LEN 32
18 #define ADF_CONST_TABLE_SIZE 1024
19 #define ADF_ADMIN_POLL_DELAY_US 20
20 #define ADF_ADMIN_POLL_TIMEOUT_US (5 * USEC_PER_SEC)
21 #define ADF_ONE_AE 1
22
23 static const u8 const_tab[1024] __aligned(1024) = {
24 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
25 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
26 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
27 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
28 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
29 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
30 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00,
31 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
32 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00,
33 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01,
34 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
35 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00,
36 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13,
37 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00,
38 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00,
39 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
40 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
41 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
42 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
43 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
44 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
45 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
46 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00,
47 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
48 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76,
49 0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
50 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab,
51 0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0,
52 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
53 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
54 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e,
55 0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39,
56 0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe,
57 0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
58 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae,
59 0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f,
60 0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05,
61 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
62 0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29,
63 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17,
64 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff,
65 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c,
66 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f,
67 0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
68 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb,
69 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94,
70 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52,
71 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f,
72 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13,
73 0x7e, 0x21, 0x79, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
74 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
75 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00,
76 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x18,
77 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
78 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00,
79 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
80 0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00,
81 0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02,
82 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
83 0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00,
84 0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25,
85 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00,
86 0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00,
87 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
88 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00,
89 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01,
90 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
91 0x00, 0x2B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00,
92 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
93 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00,
94 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,
95 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
96 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
97 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
98 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
99 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
100 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
101 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
102 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
103
104 struct adf_admin_comms {
105 dma_addr_t phy_addr;
106 dma_addr_t const_tbl_addr;
107 void *virt_addr;
108 void *virt_tbl_addr;
109 void __iomem *mailbox_addr;
110 struct mutex lock; /* protects adf_admin_comms struct */
111 };
112
adf_put_admin_msg_sync(struct adf_accel_dev * accel_dev,u32 ae,void * in,void * out)113 static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae,
114 void *in, void *out)
115 {
116 int ret;
117 u32 status;
118 struct adf_admin_comms *admin = accel_dev->admin;
119 int offset = ae * ADF_ADMINMSG_LEN * 2;
120 void __iomem *mailbox = admin->mailbox_addr;
121 int mb_offset = ae * ADF_ADMIN_MAILBOX_STRIDE;
122 struct icp_qat_fw_init_admin_req *request = in;
123
124 mutex_lock(&admin->lock);
125
126 if (ADF_CSR_RD(mailbox, mb_offset) == 1) {
127 mutex_unlock(&admin->lock);
128 return -EAGAIN;
129 }
130
131 memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN);
132 ADF_CSR_WR(mailbox, mb_offset, 1);
133
134 ret = read_poll_timeout(ADF_CSR_RD, status, status == 0,
135 ADF_ADMIN_POLL_DELAY_US,
136 ADF_ADMIN_POLL_TIMEOUT_US, true,
137 mailbox, mb_offset);
138 if (ret < 0) {
139 /* Response timeout */
140 dev_err(&GET_DEV(accel_dev),
141 "Failed to send admin msg %d to accelerator %d\n",
142 request->cmd_id, ae);
143 } else {
144 /* Response received from admin message, we can now
145 * make response data available in "out" parameter.
146 */
147 memcpy(out, admin->virt_addr + offset +
148 ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN);
149 }
150
151 mutex_unlock(&admin->lock);
152 return ret;
153 }
154
adf_send_admin(struct adf_accel_dev * accel_dev,struct icp_qat_fw_init_admin_req * req,struct icp_qat_fw_init_admin_resp * resp,const unsigned long ae_mask)155 static int adf_send_admin(struct adf_accel_dev *accel_dev,
156 struct icp_qat_fw_init_admin_req *req,
157 struct icp_qat_fw_init_admin_resp *resp,
158 const unsigned long ae_mask)
159 {
160 u32 ae;
161
162 for_each_set_bit(ae, &ae_mask, ICP_QAT_HW_AE_DELIMITER)
163 if (adf_put_admin_msg_sync(accel_dev, ae, req, resp) ||
164 resp->status)
165 return -EFAULT;
166
167 return 0;
168 }
169
adf_init_ae(struct adf_accel_dev * accel_dev)170 static int adf_init_ae(struct adf_accel_dev *accel_dev)
171 {
172 struct icp_qat_fw_init_admin_req req;
173 struct icp_qat_fw_init_admin_resp resp;
174 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
175 u32 ae_mask = hw_device->ae_mask;
176
177 memset(&req, 0, sizeof(req));
178 memset(&resp, 0, sizeof(resp));
179 req.cmd_id = ICP_QAT_FW_INIT_AE;
180
181 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
182 }
183
adf_set_fw_constants(struct adf_accel_dev * accel_dev)184 static int adf_set_fw_constants(struct adf_accel_dev *accel_dev)
185 {
186 struct icp_qat_fw_init_admin_req req;
187 struct icp_qat_fw_init_admin_resp resp;
188 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
189 u32 ae_mask = hw_device->admin_ae_mask ?: hw_device->ae_mask;
190
191 memset(&req, 0, sizeof(req));
192 memset(&resp, 0, sizeof(resp));
193 req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG;
194
195 req.init_cfg_sz = ADF_CONST_TABLE_SIZE;
196 req.init_cfg_ptr = accel_dev->admin->const_tbl_addr;
197
198 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
199 }
200
adf_get_fw_timestamp(struct adf_accel_dev * accel_dev,u64 * timestamp)201 int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp)
202 {
203 struct icp_qat_fw_init_admin_req req = { };
204 struct icp_qat_fw_init_admin_resp resp;
205 unsigned int ae_mask = ADF_ONE_AE;
206 int ret;
207
208 req.cmd_id = ICP_QAT_FW_TIMER_GET;
209 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask);
210 if (ret)
211 return ret;
212
213 *timestamp = resp.timestamp;
214 return 0;
215 }
216
adf_set_chaining(struct adf_accel_dev * accel_dev)217 static int adf_set_chaining(struct adf_accel_dev *accel_dev)
218 {
219 u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask;
220 struct icp_qat_fw_init_admin_resp resp = { };
221 struct icp_qat_fw_init_admin_req req = { };
222
223 req.cmd_id = ICP_QAT_FW_DC_CHAIN_INIT;
224
225 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
226 }
227
adf_get_dc_capabilities(struct adf_accel_dev * accel_dev,u32 * capabilities)228 static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev,
229 u32 *capabilities)
230 {
231 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
232 struct icp_qat_fw_init_admin_resp resp;
233 struct icp_qat_fw_init_admin_req req;
234 unsigned long ae_mask;
235 unsigned long ae;
236 int ret;
237
238 /* Target only service accelerator engines */
239 ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask;
240
241 memset(&req, 0, sizeof(req));
242 memset(&resp, 0, sizeof(resp));
243 req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET;
244
245 *capabilities = 0;
246 for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) {
247 ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae);
248 if (ret)
249 return ret;
250
251 *capabilities |= resp.extended_features;
252 }
253
254 return 0;
255 }
256
adf_get_ae_fw_counters(struct adf_accel_dev * accel_dev,u16 ae,u64 * reqs,u64 * resps)257 int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps)
258 {
259 struct icp_qat_fw_init_admin_resp resp = { };
260 struct icp_qat_fw_init_admin_req req = { };
261 int ret;
262
263 req.cmd_id = ICP_QAT_FW_COUNTERS_GET;
264
265 ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp);
266 if (ret || resp.status)
267 return -EFAULT;
268
269 *reqs = resp.req_rec_count;
270 *resps = resp.resp_sent_count;
271
272 return 0;
273 }
274
adf_send_admin_tim_sync(struct adf_accel_dev * accel_dev,u32 cnt)275 int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt)
276 {
277 u32 ae_mask = accel_dev->hw_device->ae_mask;
278 struct icp_qat_fw_init_admin_req req = { };
279 struct icp_qat_fw_init_admin_resp resp = { };
280
281 req.cmd_id = ICP_QAT_FW_SYNC;
282 req.int_timer_ticks = cnt;
283
284 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
285 }
286
adf_send_admin_hb_timer(struct adf_accel_dev * accel_dev,uint32_t ticks)287 int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks)
288 {
289 u32 ae_mask = accel_dev->hw_device->ae_mask;
290 struct icp_qat_fw_init_admin_req req = { };
291 struct icp_qat_fw_init_admin_resp resp;
292
293 req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET;
294 req.init_cfg_ptr = accel_dev->heartbeat->dma.phy_addr;
295 req.heartbeat_ticks = ticks;
296
297 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
298 }
299
is_dcc_enabled(struct adf_accel_dev * accel_dev)300 static bool is_dcc_enabled(struct adf_accel_dev *accel_dev)
301 {
302 char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0};
303 int ret;
304
305 ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
306 ADF_SERVICES_ENABLED, services);
307 if (ret)
308 return false;
309
310 return !strcmp(services, "dcc");
311 }
312
adf_get_fw_capabilities(struct adf_accel_dev * accel_dev,u16 * caps)313 static int adf_get_fw_capabilities(struct adf_accel_dev *accel_dev, u16 *caps)
314 {
315 u32 ae_mask = accel_dev->hw_device->admin_ae_mask;
316 struct icp_qat_fw_init_admin_resp resp = { };
317 struct icp_qat_fw_init_admin_req req = { };
318 int ret;
319
320 if (!ae_mask)
321 return 0;
322
323 req.cmd_id = ICP_QAT_FW_CAPABILITIES_GET;
324 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask);
325 if (ret)
326 return ret;
327
328 *caps = resp.fw_capabilities;
329
330 return 0;
331 }
332
adf_send_admin_rl_init(struct adf_accel_dev * accel_dev,struct icp_qat_fw_init_admin_slice_cnt * slices)333 int adf_send_admin_rl_init(struct adf_accel_dev *accel_dev,
334 struct icp_qat_fw_init_admin_slice_cnt *slices)
335 {
336 u32 ae_mask = accel_dev->hw_device->admin_ae_mask;
337 struct icp_qat_fw_init_admin_resp resp = { };
338 struct icp_qat_fw_init_admin_req req = { };
339 int ret;
340
341 req.cmd_id = ICP_QAT_FW_RL_INIT;
342
343 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask);
344 if (ret)
345 return ret;
346
347 memcpy(slices, &resp.slices, sizeof(*slices));
348
349 return 0;
350 }
351
adf_send_admin_rl_add_update(struct adf_accel_dev * accel_dev,struct icp_qat_fw_init_admin_req * req)352 int adf_send_admin_rl_add_update(struct adf_accel_dev *accel_dev,
353 struct icp_qat_fw_init_admin_req *req)
354 {
355 u32 ae_mask = accel_dev->hw_device->admin_ae_mask;
356 struct icp_qat_fw_init_admin_resp resp = { };
357
358 /*
359 * req struct filled in rl implementation. Used commands
360 * ICP_QAT_FW_RL_ADD for a new SLA
361 * ICP_QAT_FW_RL_UPDATE for update SLA
362 */
363 return adf_send_admin(accel_dev, req, &resp, ae_mask);
364 }
365
adf_send_admin_rl_delete(struct adf_accel_dev * accel_dev,u16 node_id,u8 node_type)366 int adf_send_admin_rl_delete(struct adf_accel_dev *accel_dev, u16 node_id,
367 u8 node_type)
368 {
369 u32 ae_mask = accel_dev->hw_device->admin_ae_mask;
370 struct icp_qat_fw_init_admin_resp resp = { };
371 struct icp_qat_fw_init_admin_req req = { };
372
373 req.cmd_id = ICP_QAT_FW_RL_REMOVE;
374 req.node_id = node_id;
375 req.node_type = node_type;
376
377 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
378 }
379
380 /**
381 * adf_send_admin_init() - Function sends init message to FW
382 * @accel_dev: Pointer to acceleration device.
383 *
384 * Function sends admin init message to the FW
385 *
386 * Return: 0 on success, error code otherwise.
387 */
adf_send_admin_init(struct adf_accel_dev * accel_dev)388 int adf_send_admin_init(struct adf_accel_dev *accel_dev)
389 {
390 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
391 u32 dc_capabilities = 0;
392 int ret;
393
394 ret = adf_set_fw_constants(accel_dev);
395 if (ret)
396 return ret;
397
398 if (is_dcc_enabled(accel_dev)) {
399 ret = adf_set_chaining(accel_dev);
400 if (ret)
401 return ret;
402 }
403
404 ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities);
405 if (ret) {
406 dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n");
407 return ret;
408 }
409 accel_dev->hw_device->extended_dc_capabilities = dc_capabilities;
410
411 adf_get_fw_capabilities(accel_dev, &hw_data->fw_capabilities);
412
413 return adf_init_ae(accel_dev);
414 }
415 EXPORT_SYMBOL_GPL(adf_send_admin_init);
416
417 /**
418 * adf_init_admin_pm() - Function sends PM init message to FW
419 * @accel_dev: Pointer to acceleration device.
420 * @idle_delay: QAT HW idle time before power gating is initiated.
421 * 000 - 64us
422 * 001 - 128us
423 * 010 - 256us
424 * 011 - 512us
425 * 100 - 1ms
426 * 101 - 2ms
427 * 110 - 4ms
428 * 111 - 8ms
429 *
430 * Function sends to the FW the admin init message for the PM state
431 * configuration.
432 *
433 * Return: 0 on success, error code otherwise.
434 */
adf_init_admin_pm(struct adf_accel_dev * accel_dev,u32 idle_delay)435 int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay)
436 {
437 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
438 struct icp_qat_fw_init_admin_resp resp = {0};
439 struct icp_qat_fw_init_admin_req req = {0};
440 u32 ae_mask = hw_data->admin_ae_mask;
441
442 if (!accel_dev->admin) {
443 dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n");
444 return -EFAULT;
445 }
446
447 req.cmd_id = ICP_QAT_FW_PM_STATE_CONFIG;
448 req.idle_filter = idle_delay;
449
450 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
451 }
452 EXPORT_SYMBOL_GPL(adf_init_admin_pm);
453
adf_get_pm_info(struct adf_accel_dev * accel_dev,dma_addr_t p_state_addr,size_t buff_size)454 int adf_get_pm_info(struct adf_accel_dev *accel_dev, dma_addr_t p_state_addr,
455 size_t buff_size)
456 {
457 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
458 struct icp_qat_fw_init_admin_req req = { };
459 struct icp_qat_fw_init_admin_resp resp;
460 u32 ae_mask = hw_data->admin_ae_mask;
461 int ret;
462
463 /* Query pm info via init/admin cmd */
464 if (!accel_dev->admin) {
465 dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n");
466 return -EFAULT;
467 }
468
469 req.cmd_id = ICP_QAT_FW_PM_INFO;
470 req.init_cfg_sz = buff_size;
471 req.init_cfg_ptr = p_state_addr;
472
473 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask);
474 if (ret)
475 dev_err(&GET_DEV(accel_dev),
476 "Failed to query power-management info\n");
477
478 return ret;
479 }
480
adf_get_cnv_stats(struct adf_accel_dev * accel_dev,u16 ae,u16 * err_cnt,u16 * latest_err)481 int adf_get_cnv_stats(struct adf_accel_dev *accel_dev, u16 ae, u16 *err_cnt,
482 u16 *latest_err)
483 {
484 struct icp_qat_fw_init_admin_req req = { };
485 struct icp_qat_fw_init_admin_resp resp;
486 int ret;
487
488 req.cmd_id = ICP_QAT_FW_CNV_STATS_GET;
489
490 ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp);
491 if (ret)
492 return ret;
493 if (resp.status)
494 return -EPROTONOSUPPORT;
495
496 *err_cnt = resp.error_count;
497 *latest_err = resp.latest_error;
498
499 return ret;
500 }
501
adf_send_admin_tl_start(struct adf_accel_dev * accel_dev,dma_addr_t tl_dma_addr,size_t layout_sz,u8 * rp_indexes,struct icp_qat_fw_init_admin_slice_cnt * slice_count)502 int adf_send_admin_tl_start(struct adf_accel_dev *accel_dev,
503 dma_addr_t tl_dma_addr, size_t layout_sz, u8 *rp_indexes,
504 struct icp_qat_fw_init_admin_slice_cnt *slice_count)
505 {
506 u32 ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask;
507 struct icp_qat_fw_init_admin_resp resp = { };
508 struct icp_qat_fw_init_admin_req req = { };
509 int ret;
510
511 req.cmd_id = ICP_QAT_FW_TL_START;
512 req.init_cfg_ptr = tl_dma_addr;
513 req.init_cfg_sz = layout_sz;
514
515 if (rp_indexes)
516 memcpy(&req.rp_indexes, rp_indexes, sizeof(req.rp_indexes));
517
518 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask);
519 if (ret)
520 return ret;
521
522 memcpy(slice_count, &resp.slices, sizeof(*slice_count));
523
524 return 0;
525 }
526
adf_send_admin_tl_stop(struct adf_accel_dev * accel_dev)527 int adf_send_admin_tl_stop(struct adf_accel_dev *accel_dev)
528 {
529 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
530 struct icp_qat_fw_init_admin_resp resp = { };
531 struct icp_qat_fw_init_admin_req req = { };
532 u32 ae_mask = hw_data->admin_ae_mask;
533
534 req.cmd_id = ICP_QAT_FW_TL_STOP;
535
536 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
537 }
538
adf_init_admin_comms(struct adf_accel_dev * accel_dev)539 int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
540 {
541 struct adf_admin_comms *admin;
542 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
543 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
544 struct admin_info admin_csrs_info;
545 u32 mailbox_offset, adminmsg_u, adminmsg_l;
546 void __iomem *mailbox;
547 u64 reg_val;
548
549 admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL,
550 dev_to_node(&GET_DEV(accel_dev)));
551 if (!admin)
552 return -ENOMEM;
553 admin->virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
554 &admin->phy_addr, GFP_KERNEL);
555 if (!admin->virt_addr) {
556 dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n");
557 kfree(admin);
558 return -ENOMEM;
559 }
560
561 admin->virt_tbl_addr = dma_alloc_coherent(&GET_DEV(accel_dev),
562 PAGE_SIZE,
563 &admin->const_tbl_addr,
564 GFP_KERNEL);
565 if (!admin->virt_tbl_addr) {
566 dev_err(&GET_DEV(accel_dev), "Failed to allocate const_tbl\n");
567 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
568 admin->virt_addr, admin->phy_addr);
569 kfree(admin);
570 return -ENOMEM;
571 }
572
573 memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab));
574 hw_data->get_admin_info(&admin_csrs_info);
575
576 mailbox_offset = admin_csrs_info.mailbox_offset;
577 mailbox = pmisc_addr + mailbox_offset;
578 adminmsg_u = admin_csrs_info.admin_msg_ur;
579 adminmsg_l = admin_csrs_info.admin_msg_lr;
580
581 reg_val = (u64)admin->phy_addr;
582 ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val));
583 ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val));
584
585 mutex_init(&admin->lock);
586 admin->mailbox_addr = mailbox;
587 accel_dev->admin = admin;
588 return 0;
589 }
590 EXPORT_SYMBOL_GPL(adf_init_admin_comms);
591
adf_exit_admin_comms(struct adf_accel_dev * accel_dev)592 void adf_exit_admin_comms(struct adf_accel_dev *accel_dev)
593 {
594 struct adf_admin_comms *admin = accel_dev->admin;
595
596 if (!admin)
597 return;
598
599 if (admin->virt_addr)
600 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
601 admin->virt_addr, admin->phy_addr);
602 if (admin->virt_tbl_addr)
603 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
604 admin->virt_tbl_addr, admin->const_tbl_addr);
605
606 mutex_destroy(&admin->lock);
607 kfree(admin);
608 accel_dev->admin = NULL;
609 }
610 EXPORT_SYMBOL_GPL(adf_exit_admin_comms);
611