1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2/* Copyright(c) 2021 Intel Corporation */
3#include <linux/delay.h>
4#include <linux/iopoll.h>
5#include <linux/mutex.h>
6#include <linux/types.h>
7#include "adf_accel_devices.h"
8#include "adf_common_drv.h"
9#include "adf_gen2_pfvf.h"
10#include "adf_pfvf_msg.h"
11#include "adf_pfvf_pf_proto.h"
12#include "adf_pfvf_vf_proto.h"
13#include "adf_pfvf_utils.h"
14
15 /* VF2PF interrupts */
16#define ADF_GEN2_VF_MSK 0xFFFF
17#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9)
18#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & ADF_GEN2_VF_MSK) << 9)
19
20#define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04))
21#define ADF_GEN2_VF_PF2VF_OFFSET 0x200
22
23#define ADF_GEN2_CSR_IN_USE 0x6AC2
24#define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE
25
26enum gen2_csr_pos {
27 ADF_GEN2_CSR_PF2VF_OFFSET = 0,
28 ADF_GEN2_CSR_VF2PF_OFFSET = 16,
29};
30
31#define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2
32#define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F
33#define ADF_PFVF_GEN2_MSGDATA_SHIFT 6
34#define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF
35
36static const struct pfvf_csr_format csr_gen2_fmt = {
37 { ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK },
38 { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK },
39};
40
41#define ADF_PFVF_MSG_RETRY_DELAY 5
42#define ADF_PFVF_MSG_MAX_RETRIES 3
43
44static u32 adf_gen2_pf_get_pfvf_offset(u32 i)
45{
46 return ADF_GEN2_PF_PF2VF_OFFSET(i);
47}
48
49static u32 adf_gen2_vf_get_pfvf_offset(u32 i)
50{
51 return ADF_GEN2_VF_PF2VF_OFFSET;
52}
53
54static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
55{
56 /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */
57 if (vf_mask & ADF_GEN2_VF_MSK) {
58 u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
59 & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
60 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
61 }
62}
63
64static void adf_gen2_disable_all_vf2pf_interrupts(void __iomem *pmisc_addr)
65{
66 /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */
67 u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
68 | ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK);
69 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
70}
71
72static u32 adf_gen2_disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr)
73{
74 u32 sources, disabled, pending;
75 u32 errsou3, errmsk3;
76
77 /* Get the interrupt sources triggered by VFs */
78 errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3);
79 sources = ADF_GEN2_ERR_REG_VF2PF(errsou3);
80
81 if (!sources)
82 return 0;
83
84 /* Get the already disabled interrupts */
85 errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3);
86 disabled = ADF_GEN2_ERR_REG_VF2PF(errmsk3);
87
88 pending = sources & ~disabled;
89 if (!pending)
90 return 0;
91
92 /* Due to HW limitations, when disabling the interrupts, we can't
93 * just disable the requested sources, as this would lead to missed
94 * interrupts if ERRSOU3 changes just before writing to ERRMSK3.
95 * To work around it, disable all and re-enable only the sources that
96 * are not in vf_mask and were not already disabled. Re-enabling will
97 * trigger a new interrupt for the sources that have changed in the
98 * meantime, if any.
99 */
100 errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK);
101 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3);
102
103 /* Update only section of errmsk3 related to VF2PF */
104 errmsk3 &= ~ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK);
105 errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(sources | disabled);
106 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3);
107
108 /* Return the sources of the (new) interrupt(s) */
109 return pending;
110}
111
112static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset)
113{
114 return ADF_PFVF_INT << offset;
115}
116
117static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset)
118{
119 return (csr_msg & 0xFFFF) << offset;
120}
121
122static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset)
123{
124 return (csr_val >> offset) & 0xFFFF;
125}
126
127static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset)
128{
129 return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE;
130}
131
132static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset)
133{
134 *msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset);
135}
136
137static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset)
138{
139 *msg |= (ADF_GEN2_CSR_IN_USE << offset);
140}
141
142static bool is_legacy_user_pfvf_message(u32 msg)
143{
144 return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM);
145}
146
147static bool is_pf2vf_notification(u8 msg_type)
148{
149 switch (msg_type) {
150 case ADF_PF2VF_MSGTYPE_RESTARTING:
151 return true;
152 default:
153 return false;
154 }
155}
156
157static bool is_vf2pf_notification(u8 msg_type)
158{
159 switch (msg_type) {
160 case ADF_VF2PF_MSGTYPE_INIT:
161 case ADF_VF2PF_MSGTYPE_SHUTDOWN:
162 return true;
163 default:
164 return false;
165 }
166}
167
168struct pfvf_gen2_params {
169 u32 pfvf_offset;
170 struct mutex *csr_lock; /* lock preventing concurrent access of CSR */
171 enum gen2_csr_pos local_offset;
172 enum gen2_csr_pos remote_offset;
173 bool (*is_notification_message)(u8 msg_type);
174 u8 compat_ver;
175};
176
177static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev,
178 struct pfvf_message msg,
179 struct pfvf_gen2_params *params)
180{
181 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
182 enum gen2_csr_pos remote_offset = params->remote_offset;
183 enum gen2_csr_pos local_offset = params->local_offset;
184 unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES;
185 struct mutex *lock = params->csr_lock;
186 u32 pfvf_offset = params->pfvf_offset;
187 u32 int_bit;
188 u32 csr_val;
189 u32 csr_msg;
190 int ret;
191
192 /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This
193 * allows us to build and read messages as if they where all 0 based.
194 * However, send and receive are in a single shared 32 bits register,
195 * so we need to shift and/or mask the message half before decoding
196 * it and after encoding it. Which one to shift depends on the
197 * direction.
198 */
199
200 int_bit = gen2_csr_get_int_bit(offset: local_offset);
201
202 csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, fmt: &csr_gen2_fmt);
203 if (unlikely(!csr_msg))
204 return -EINVAL;
205
206 /* Prepare for CSR format, shifting the wire message in place and
207 * setting the in use pattern
208 */
209 csr_msg = gen2_csr_msg_to_position(csr_msg, offset: local_offset);
210 gen2_csr_set_in_use(msg: &csr_msg, offset: remote_offset);
211
212 mutex_lock(lock);
213
214start:
215 /* Check if the PFVF CSR is in use by remote function */
216 csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
217 if (gen2_csr_is_in_use(msg: csr_val, offset: local_offset)) {
218 dev_dbg(&GET_DEV(accel_dev),
219 "PFVF CSR in use by remote function\n");
220 goto retry;
221 }
222
223 /* Attempt to get ownership of the PFVF CSR */
224 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit);
225
226 /* Wait for confirmation from remote func it received the message */
227 ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit),
228 ADF_PFVF_MSG_ACK_DELAY_US,
229 ADF_PFVF_MSG_ACK_MAX_DELAY_US,
230 true, pmisc_addr, pfvf_offset);
231 if (unlikely(ret < 0)) {
232 dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
233 csr_val &= ~int_bit;
234 }
235
236 /* For fire-and-forget notifications, the receiver does not clear
237 * the in-use pattern. This is used to detect collisions.
238 */
239 if (params->is_notification_message(msg.type) && csr_val != csr_msg) {
240 /* Collision must have overwritten the message */
241 dev_err(&GET_DEV(accel_dev),
242 "Collision on notification - PFVF CSR overwritten by remote function\n");
243 goto retry;
244 }
245
246 /* If the far side did not clear the in-use pattern it is either
247 * 1) Notification - message left intact to detect collision
248 * 2) Older protocol (compatibility version < 3) on the far side
249 * where the sender is responsible for clearing the in-use
250 * pattern after the received has acknowledged receipt.
251 * In either case, clear the in-use pattern now.
252 */
253 if (gen2_csr_is_in_use(msg: csr_val, offset: remote_offset)) {
254 gen2_csr_clear_in_use(msg: &csr_val, offset: remote_offset);
255 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
256 }
257
258out:
259 mutex_unlock(lock);
260 return ret;
261
262retry:
263 if (--retries) {
264 msleep(ADF_PFVF_MSG_RETRY_DELAY);
265 goto start;
266 } else {
267 ret = -EBUSY;
268 goto out;
269 }
270}
271
272static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev,
273 struct pfvf_gen2_params *params)
274{
275 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
276 enum gen2_csr_pos remote_offset = params->remote_offset;
277 enum gen2_csr_pos local_offset = params->local_offset;
278 u32 pfvf_offset = params->pfvf_offset;
279 struct pfvf_message msg = { 0 };
280 u32 int_bit;
281 u32 csr_val;
282 u16 csr_msg;
283
284 int_bit = gen2_csr_get_int_bit(offset: local_offset);
285
286 /* Read message */
287 csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
288 if (!(csr_val & int_bit)) {
289 dev_info(&GET_DEV(accel_dev),
290 "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val);
291 return msg;
292 }
293
294 /* Extract the message from the CSR */
295 csr_msg = gen2_csr_msg_from_position(csr_val, offset: local_offset);
296
297 /* Ignore legacy non-system (non-kernel) messages */
298 if (unlikely(is_legacy_user_pfvf_message(csr_msg))) {
299 dev_dbg(&GET_DEV(accel_dev),
300 "Ignored non-system message (0x%.8x);\n", csr_val);
301 /* Because this must be a legacy message, the far side
302 * must clear the in-use pattern, so don't do it.
303 */
304 return msg;
305 }
306
307 /* Return the pfvf_message format */
308 msg = adf_pfvf_message_of(accel_dev, raw_msg: csr_msg, fmt: &csr_gen2_fmt);
309
310 /* The in-use pattern is not cleared for notifications (so that
311 * it can be used for collision detection) or older implementations
312 */
313 if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK &&
314 !params->is_notification_message(msg.type))
315 gen2_csr_clear_in_use(msg: &csr_val, offset: remote_offset);
316
317 /* To ACK, clear the INT bit */
318 csr_val &= ~int_bit;
319 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
320
321 return msg;
322}
323
324static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
325 u32 pfvf_offset, struct mutex *csr_lock)
326{
327 struct pfvf_gen2_params params = {
328 .csr_lock = csr_lock,
329 .pfvf_offset = pfvf_offset,
330 .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
331 .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
332 .is_notification_message = is_pf2vf_notification,
333 };
334
335 return adf_gen2_pfvf_send(accel_dev, msg, params: &params);
336}
337
338static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
339 u32 pfvf_offset, struct mutex *csr_lock)
340{
341 struct pfvf_gen2_params params = {
342 .csr_lock = csr_lock,
343 .pfvf_offset = pfvf_offset,
344 .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
345 .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
346 .is_notification_message = is_vf2pf_notification,
347 };
348
349 return adf_gen2_pfvf_send(accel_dev, msg, params: &params);
350}
351
352static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev,
353 u32 pfvf_offset, u8 compat_ver)
354{
355 struct pfvf_gen2_params params = {
356 .pfvf_offset = pfvf_offset,
357 .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
358 .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
359 .is_notification_message = is_pf2vf_notification,
360 .compat_ver = compat_ver,
361 };
362
363 return adf_gen2_pfvf_recv(accel_dev, params: &params);
364}
365
366static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev,
367 u32 pfvf_offset, u8 compat_ver)
368{
369 struct pfvf_gen2_params params = {
370 .pfvf_offset = pfvf_offset,
371 .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
372 .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
373 .is_notification_message = is_vf2pf_notification,
374 .compat_ver = compat_ver,
375 };
376
377 return adf_gen2_pfvf_recv(accel_dev, params: &params);
378}
379
380void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
381{
382 pfvf_ops->enable_comms = adf_enable_pf2vf_comms;
383 pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset;
384 pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset;
385 pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
386 pfvf_ops->disable_all_vf2pf_interrupts = adf_gen2_disable_all_vf2pf_interrupts;
387 pfvf_ops->disable_pending_vf2pf_interrupts = adf_gen2_disable_pending_vf2pf_interrupts;
388 pfvf_ops->send_msg = adf_gen2_pf2vf_send;
389 pfvf_ops->recv_msg = adf_gen2_vf2pf_recv;
390}
391EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops);
392
393void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
394{
395 pfvf_ops->enable_comms = adf_enable_vf2pf_comms;
396 pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset;
397 pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset;
398 pfvf_ops->send_msg = adf_gen2_vf2pf_send;
399 pfvf_ops->recv_msg = adf_gen2_pf2vf_recv;
400}
401EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops);
402

source code of linux/drivers/crypto/intel/qat/qat_common/adf_gen2_pfvf.c