1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include "pvr_ccb.h"
5#include "pvr_cccb.h"
6#include "pvr_device.h"
7#include "pvr_gem.h"
8#include "pvr_hwrt.h"
9
10#include <linux/compiler.h>
11#include <linux/delay.h>
12#include <linux/jiffies.h>
13#include <linux/mutex.h>
14#include <linux/types.h>
15
16static __always_inline u32
17get_ccb_space(u32 w_off, u32 r_off, u32 ccb_size)
18{
19 return (((r_off) - (w_off)) + ((ccb_size) - 1)) & ((ccb_size) - 1);
20}
21
22static void
23cccb_ctrl_init(void *cpu_ptr, void *priv)
24{
25 struct rogue_fwif_cccb_ctl *ctrl = cpu_ptr;
26 struct pvr_cccb *pvr_cccb = priv;
27
28 WRITE_ONCE(ctrl->write_offset, 0);
29 WRITE_ONCE(ctrl->read_offset, 0);
30 WRITE_ONCE(ctrl->dep_offset, 0);
31 WRITE_ONCE(ctrl->wrap_mask, pvr_cccb->wrap_mask);
32}
33
34/**
35 * pvr_cccb_init() - Initialise a Client CCB
36 * @pvr_dev: Device pointer.
37 * @pvr_cccb: Pointer to Client CCB structure to initialise.
38 * @size_log2: Log2 size of Client CCB in bytes.
39 * @name: Name of owner of Client CCB. Used for fence context.
40 *
41 * Return:
42 * * Zero on success, or
43 * * Any error code returned by pvr_fw_object_create_and_map().
44 */
45int
46pvr_cccb_init(struct pvr_device *pvr_dev, struct pvr_cccb *pvr_cccb,
47 u32 size_log2, const char *name)
48{
49 size_t size = 1 << size_log2;
50 int err;
51
52 pvr_cccb->size = size;
53 pvr_cccb->write_offset = 0;
54 pvr_cccb->wrap_mask = size - 1;
55
56 /*
57 * Map CCCB and control structure as uncached, so we don't have to flush
58 * CPU cache repeatedly when polling for space.
59 */
60 pvr_cccb->ctrl = pvr_fw_object_create_and_map(pvr_dev, size: sizeof(*pvr_cccb->ctrl),
61 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
62 init: cccb_ctrl_init, init_priv: pvr_cccb,
63 pvr_obj_out: &pvr_cccb->ctrl_obj);
64 if (IS_ERR(ptr: pvr_cccb->ctrl))
65 return PTR_ERR(ptr: pvr_cccb->ctrl);
66
67 pvr_cccb->cccb = pvr_fw_object_create_and_map(pvr_dev, size,
68 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
69 NULL, NULL, pvr_obj_out: &pvr_cccb->cccb_obj);
70 if (IS_ERR(ptr: pvr_cccb->cccb)) {
71 err = PTR_ERR(ptr: pvr_cccb->cccb);
72 goto err_free_ctrl;
73 }
74
75 pvr_fw_object_get_fw_addr(fw_obj: pvr_cccb->ctrl_obj, fw_addr_out: &pvr_cccb->ctrl_fw_addr);
76 pvr_fw_object_get_fw_addr(fw_obj: pvr_cccb->cccb_obj, fw_addr_out: &pvr_cccb->cccb_fw_addr);
77
78 return 0;
79
80err_free_ctrl:
81 pvr_fw_object_unmap_and_destroy(fw_obj: pvr_cccb->ctrl_obj);
82
83 return err;
84}
85
86/**
87 * pvr_cccb_fini() - Release Client CCB structure
88 * @pvr_cccb: Client CCB to release.
89 */
90void
91pvr_cccb_fini(struct pvr_cccb *pvr_cccb)
92{
93 pvr_fw_object_unmap_and_destroy(fw_obj: pvr_cccb->cccb_obj);
94 pvr_fw_object_unmap_and_destroy(fw_obj: pvr_cccb->ctrl_obj);
95}
96
97/**
98 * pvr_cccb_cmdseq_fits() - Check if a command sequence fits in the CCCB
99 * @pvr_cccb: Target Client CCB.
100 * @size: Size of the command sequence.
101 *
102 * Check if a command sequence fits in the CCCB we have at hand.
103 *
104 * Return:
105 * * true if the command sequence fits in the CCCB, or
106 * * false otherwise.
107 */
108bool pvr_cccb_cmdseq_fits(struct pvr_cccb *pvr_cccb, size_t size)
109{
110 struct rogue_fwif_cccb_ctl *ctrl = pvr_cccb->ctrl;
111 u32 read_offset, remaining;
112 bool fits = false;
113
114 read_offset = READ_ONCE(ctrl->read_offset);
115 remaining = pvr_cccb->size - pvr_cccb->write_offset;
116
117 /* Always ensure we have enough room for a padding command at the end of the CCCB.
118 * If our command sequence does not fit, reserve the remaining space for a padding
119 * command.
120 */
121 if (size + PADDING_COMMAND_SIZE > remaining)
122 size += remaining;
123
124 if (get_ccb_space(w_off: pvr_cccb->write_offset, r_off: read_offset, ccb_size: pvr_cccb->size) >= size)
125 fits = true;
126
127 return fits;
128}
129
130/**
131 * pvr_cccb_write_command_with_header() - Write a command + command header to a
132 * Client CCB
133 * @pvr_cccb: Target Client CCB.
134 * @cmd_type: Client CCB command type. Must be one of %ROGUE_FWIF_CCB_CMD_TYPE_*.
135 * @cmd_size: Size of command in bytes.
136 * @cmd_data: Pointer to command to write.
137 * @ext_job_ref: External job reference.
138 * @int_job_ref: Internal job reference.
139 *
140 * Caller must make sure there's enough space in CCCB to queue this command. This
141 * can be done by calling pvr_cccb_cmdseq_fits().
142 *
143 * This function is not protected by any lock. The caller must ensure there's
144 * no concurrent caller, which should be guaranteed by the drm_sched model (job
145 * submission is serialized in drm_sched_main()).
146 */
147void
148pvr_cccb_write_command_with_header(struct pvr_cccb *pvr_cccb, u32 cmd_type, u32 cmd_size,
149 void *cmd_data, u32 ext_job_ref, u32 int_job_ref)
150{
151 u32 sz_with_hdr = pvr_cccb_get_size_of_cmd_with_hdr(cmd_size);
152 struct rogue_fwif_ccb_cmd_header cmd_header = {
153 .cmd_type = cmd_type,
154 .cmd_size = ALIGN(cmd_size, 8),
155 .ext_job_ref = ext_job_ref,
156 .int_job_ref = int_job_ref,
157 };
158 struct rogue_fwif_cccb_ctl *ctrl = pvr_cccb->ctrl;
159 u32 remaining = pvr_cccb->size - pvr_cccb->write_offset;
160 u32 required_size, cccb_space, read_offset;
161
162 /*
163 * Always ensure we have enough room for a padding command at the end of
164 * the CCCB.
165 */
166 if (remaining < sz_with_hdr + PADDING_COMMAND_SIZE) {
167 /*
168 * Command would need to wrap, so we need to pad the remainder
169 * of the CCCB.
170 */
171 required_size = sz_with_hdr + remaining;
172 } else {
173 required_size = sz_with_hdr;
174 }
175
176 read_offset = READ_ONCE(ctrl->read_offset);
177 cccb_space = get_ccb_space(w_off: pvr_cccb->write_offset, r_off: read_offset, ccb_size: pvr_cccb->size);
178 if (WARN_ON(cccb_space < required_size))
179 return;
180
181 if (required_size != sz_with_hdr) {
182 /* Add padding command */
183 struct rogue_fwif_ccb_cmd_header pad_cmd = {
184 .cmd_type = ROGUE_FWIF_CCB_CMD_TYPE_PADDING,
185 .cmd_size = remaining - sizeof(pad_cmd),
186 };
187
188 memcpy(&pvr_cccb->cccb[pvr_cccb->write_offset], &pad_cmd, sizeof(pad_cmd));
189 pvr_cccb->write_offset = 0;
190 }
191
192 memcpy(&pvr_cccb->cccb[pvr_cccb->write_offset], &cmd_header, sizeof(cmd_header));
193 memcpy(&pvr_cccb->cccb[pvr_cccb->write_offset + sizeof(cmd_header)], cmd_data, cmd_size);
194 pvr_cccb->write_offset += sz_with_hdr;
195}
196
197static void fill_cmd_kick_data(struct pvr_cccb *cccb, u32 ctx_fw_addr,
198 struct pvr_hwrt_data *hwrt,
199 struct rogue_fwif_kccb_cmd_kick_data *k)
200{
201 k->context_fw_addr = ctx_fw_addr;
202 k->client_woff_update = cccb->write_offset;
203 k->client_wrap_mask_update = cccb->wrap_mask;
204
205 if (hwrt) {
206 u32 cleanup_state_offset = offsetof(struct rogue_fwif_hwrtdata, cleanup_state);
207
208 pvr_fw_object_get_fw_addr_offset(fw_obj: hwrt->fw_obj, offset: cleanup_state_offset,
209 fw_addr_out: &k->cleanup_ctl_fw_addr[k->num_cleanup_ctl++]);
210 }
211}
212
213/**
214 * pvr_cccb_send_kccb_kick: Send KCCB kick to trigger command processing
215 * @pvr_dev: Device pointer.
216 * @pvr_cccb: Pointer to CCCB to process.
217 * @cctx_fw_addr: FW virtual address for context owning this Client CCB.
218 * @hwrt: HWRT data set associated with this kick. May be %NULL.
219 *
220 * You must call pvr_kccb_reserve_slot() and wait for the returned fence to
221 * signal (if this function didn't return NULL) before calling
222 * pvr_cccb_send_kccb_kick().
223 */
224void
225pvr_cccb_send_kccb_kick(struct pvr_device *pvr_dev,
226 struct pvr_cccb *pvr_cccb, u32 cctx_fw_addr,
227 struct pvr_hwrt_data *hwrt)
228{
229 struct rogue_fwif_kccb_cmd cmd_kick = {
230 .cmd_type = ROGUE_FWIF_KCCB_CMD_KICK,
231 };
232
233 fill_cmd_kick_data(cccb: pvr_cccb, ctx_fw_addr: cctx_fw_addr, hwrt, k: &cmd_kick.cmd_data.cmd_kick_data);
234
235 /* Make sure the writes to the CCCB are flushed before sending the KICK. */
236 wmb();
237
238 pvr_kccb_send_cmd_reserved_powered(pvr_dev, cmd: &cmd_kick, NULL);
239}
240
241void
242pvr_cccb_send_kccb_combined_kick(struct pvr_device *pvr_dev,
243 struct pvr_cccb *geom_cccb,
244 struct pvr_cccb *frag_cccb,
245 u32 geom_ctx_fw_addr,
246 u32 frag_ctx_fw_addr,
247 struct pvr_hwrt_data *hwrt,
248 bool frag_is_pr)
249{
250 struct rogue_fwif_kccb_cmd cmd_kick = {
251 .cmd_type = ROGUE_FWIF_KCCB_CMD_COMBINED_GEOM_FRAG_KICK,
252 };
253
254 fill_cmd_kick_data(cccb: geom_cccb, ctx_fw_addr: geom_ctx_fw_addr, hwrt,
255 k: &cmd_kick.cmd_data.combined_geom_frag_cmd_kick_data.geom_cmd_kick_data);
256
257 /* If this is a partial-render job, we don't attach resources to cleanup-ctl array,
258 * because the resources are already retained by the geometry job.
259 */
260 fill_cmd_kick_data(cccb: frag_cccb, ctx_fw_addr: frag_ctx_fw_addr, hwrt: frag_is_pr ? NULL : hwrt,
261 k: &cmd_kick.cmd_data.combined_geom_frag_cmd_kick_data.frag_cmd_kick_data);
262
263 /* Make sure the writes to the CCCB are flushed before sending the KICK. */
264 wmb();
265
266 pvr_kccb_send_cmd_reserved_powered(pvr_dev, cmd: &cmd_kick, NULL);
267}
268

source code of linux/drivers/gpu/drm/imagination/pvr_cccb.c