1	/*
2	* SPDX-License-Identifier: MIT
3	*
4	* Copyright © 2017 Intel Corporation
5	*/
6
7	#include <linux/prime_numbers.h>
8	#include <linux/string_helpers.h>
9
10	#include "gem/i915_gem_internal.h"
11	#include "gem/i915_gem_pm.h"
12	#include "gt/intel_engine_pm.h"
13	#include "gt/intel_engine_regs.h"
14	#include "gt/intel_gt.h"
15	#include "gt/intel_gt_requests.h"
16	#include "gt/intel_reset.h"
17	#include "i915_selftest.h"
18
19	#include "gem/selftests/igt_gem_utils.h"
20	#include "selftests/i915_random.h"
21	#include "selftests/igt_flush_test.h"
22	#include "selftests/igt_live_test.h"
23	#include "selftests/igt_reset.h"
24	#include "selftests/igt_spinner.h"
25	#include "selftests/mock_drm.h"
26	#include "selftests/mock_gem_device.h"
27
28	#include "huge_gem_object.h"
29	#include "igt_gem_utils.h"
30
31	#define DW_PER_PAGE (PAGE_SIZE / sizeof(u32))
32
33	static int live_nop_switch(void *arg)
34	{
35	const unsigned int nctx = 1024;
36	struct drm_i915_private *i915 = arg;
37	struct intel_engine_cs *engine;
38	struct i915_gem_context **ctx;
39	struct igt_live_test t;
40	struct file *file;
41	unsigned long n;
42	int err = -ENODEV;
43
44	/*
45	* Create as many contexts as we can feasibly get away with
46	* and check we can switch between them rapidly.
47	*
48	* Serves as very simple stress test for submission and HW switching
49	* between contexts.
50	*/
51
52	if (!DRIVER_CAPS(i915)->has_logical_contexts)
53	return 0;
54
55	file = mock_file(i915);
56	if (IS_ERR(ptr: file))
57	return PTR_ERR(ptr: file);
58
59	ctx = kcalloc(nctx, sizeof(*ctx), GFP_KERNEL);
60	if (!ctx) {
61	err = -ENOMEM;
62	goto out_file;
63	}
64
65	for (n = 0; n < nctx; n++) {
66	ctx[n] = live_context(i915, file);
67	if (IS_ERR(ptr: ctx[n])) {
68	err = PTR_ERR(ptr: ctx[n]);
69	goto out_ctx;
70	}
71	}
72
73	for_each_uabi_engine(engine, i915) {
74	struct i915_request *rq = NULL;
75	unsigned long end_time, prime;
76	ktime_t times[2] = {};
77
78	times[0] = ktime_get_raw();
79	for (n = 0; n < nctx; n++) {
80	struct i915_request *this;
81
82	this = igt_request_alloc(ctx: ctx[n], engine);
83	if (IS_ERR(ptr: this)) {
84	err = PTR_ERR(ptr: this);
85	goto out_ctx;
86	}
87	if (rq) {
88	i915_request_await_dma_fence(rq: this, fence: &rq->fence);
89	i915_request_put(rq);
90	}
91	rq = i915_request_get(rq: this);
92	i915_request_add(rq: this);
93	}
94	if (i915_request_wait(rq, flags: 0, timeout: 10 * HZ) < 0) {
95	pr_err("Failed to populated %d contexts\n", nctx);
96	intel_gt_set_wedged(gt: engine->gt);
97	i915_request_put(rq);
98	err = -EIO;
99	goto out_ctx;
100	}
101	i915_request_put(rq);
102
103	times[1] = ktime_get_raw();
104
105	pr_info("Populated %d contexts on %s in %lluns\n",
106	nctx, engine->name, ktime_to_ns(times[1] - times[0]));
107
108	err = igt_live_test_begin(t: &t, i915, func: __func__, name: engine->name);
109	if (err)
110	goto out_ctx;
111
112	end_time = jiffies + i915_selftest.timeout_jiffies;
113	for_each_prime_number_from(prime, 2, 8192) {
114	times[1] = ktime_get_raw();
115
116	rq = NULL;
117	for (n = 0; n < prime; n++) {
118	struct i915_request *this;
119
120	this = igt_request_alloc(ctx: ctx[n % nctx], engine);
121	if (IS_ERR(ptr: this)) {
122	err = PTR_ERR(ptr: this);
123	goto out_ctx;
124	}
125
126	if (rq) { /* Force submission order */
127	i915_request_await_dma_fence(rq: this, fence: &rq->fence);
128	i915_request_put(rq);
129	}
130
131	/*
132	* This space is left intentionally blank.
133	*
134	* We do not actually want to perform any
135	* action with this request, we just want
136	* to measure the latency in allocation
137	* and submission of our breadcrumbs -
138	* ensuring that the bare request is sufficient
139	* for the system to work (i.e. proper HEAD
140	* tracking of the rings, interrupt handling,
141	* etc). It also gives us the lowest bounds
142	* for latency.
143	*/
144
145	rq = i915_request_get(rq: this);
146	i915_request_add(rq: this);
147	}
148	GEM_BUG_ON(!rq);
149	if (i915_request_wait(rq, flags: 0, HZ / 5) < 0) {
150	pr_err("Switching between %ld contexts timed out\n",
151	prime);
152	intel_gt_set_wedged(gt: engine->gt);
153	i915_request_put(rq);
154	break;
155	}
156	i915_request_put(rq);
157
158	times[1] = ktime_sub(ktime_get_raw(), times[1]);
159	if (prime == 2)
160	times[0] = times[1];
161
162	if (__igt_timeout(timeout: end_time, NULL))
163	break;
164	}
165
166	err = igt_live_test_end(t: &t);
167	if (err)
168	goto out_ctx;
169
170	pr_info("Switch latencies on %s: 1 = %lluns, %lu = %lluns\n",
171	engine->name,
172	ktime_to_ns(times[0]),
173	prime - 1, div64_u64(ktime_to_ns(times[1]), prime - 1));
174	}
175
176	out_ctx:
177	kfree(objp: ctx);
178	out_file:
179	fput(file);
180	return err;
181	}
182
183	struct parallel_switch {
184	struct kthread_worker *worker;
185	struct kthread_work work;
186	struct intel_context *ce[2];
187	int result;
188	};
189
190	static void __live_parallel_switch1(struct kthread_work *work)
191	{
192	struct parallel_switch *arg =
193	container_of(work, typeof(*arg), work);
194	IGT_TIMEOUT(end_time);
195	unsigned long count;
196
197	count = 0;
198	arg->result = 0;
199	do {
200	struct i915_request *rq = NULL;
201	int n;
202
203	for (n = 0; !arg->result && n < ARRAY_SIZE(arg->ce); n++) {
204	struct i915_request *prev = rq;
205
206	rq = i915_request_create(ce: arg->ce[n]);
207	if (IS_ERR(ptr: rq)) {
208	i915_request_put(rq: prev);
209	arg->result = PTR_ERR(ptr: rq);
210	break;
211	}
212
213	i915_request_get(rq);
214	if (prev) {
215	arg->result =
216	i915_request_await_dma_fence(rq,
217	fence: &prev->fence);
218	i915_request_put(rq: prev);
219	}
220
221	i915_request_add(rq);
222	}
223
224	if (IS_ERR_OR_NULL(ptr: rq))
225	break;
226
227	if (i915_request_wait(rq, flags: 0, HZ) < 0)
228	arg->result = -ETIME;
229
230	i915_request_put(rq);
231
232	count++;
233	} while (!arg->result && !__igt_timeout(timeout: end_time, NULL));
234
235	pr_info("%s: %lu switches (sync) <%d>\n",
236	arg->ce[0]->engine->name, count, arg->result);
237	}
238
239	static void __live_parallel_switchN(struct kthread_work *work)
240	{
241	struct parallel_switch *arg =
242	container_of(work, typeof(*arg), work);
243	struct i915_request *rq = NULL;
244	IGT_TIMEOUT(end_time);
245	unsigned long count;
246	int n;
247
248	count = 0;
249	arg->result = 0;
250	do {
251	for (n = 0; !arg->result && n < ARRAY_SIZE(arg->ce); n++) {
252	struct i915_request *prev = rq;
253
254	rq = i915_request_create(ce: arg->ce[n]);
255	if (IS_ERR(ptr: rq)) {
256	i915_request_put(rq: prev);
257	arg->result = PTR_ERR(ptr: rq);
258	break;
259	}
260
261	i915_request_get(rq);
262	if (prev) {
263	arg->result =
264	i915_request_await_dma_fence(rq,
265	fence: &prev->fence);
266	i915_request_put(rq: prev);
267	}
268
269	i915_request_add(rq);
270	}
271
272	count++;
273	} while (!arg->result && !__igt_timeout(timeout: end_time, NULL));
274
275	if (!IS_ERR_OR_NULL(ptr: rq))
276	i915_request_put(rq);
277
278	pr_info("%s: %lu switches (many) <%d>\n",
279	arg->ce[0]->engine->name, count, arg->result);
280	}
281
282	static int live_parallel_switch(void *arg)
283	{
284	struct drm_i915_private *i915 = arg;
285	static void (* const func[])(struct kthread_work *) = {
286	__live_parallel_switch1,
287	__live_parallel_switchN,
288	NULL,
289	};
290	struct parallel_switch *data = NULL;
291	struct i915_gem_engines *engines;
292	struct i915_gem_engines_iter it;
293	void (* const *fn)(struct kthread_work *);
294	struct i915_gem_context *ctx;
295	struct intel_context *ce;
296	struct file *file;
297	int n, m, count;
298	int err = 0;
299
300	/*
301	* Check we can process switches on all engines simultaneously.
302	*/
303
304	if (!DRIVER_CAPS(i915)->has_logical_contexts)
305	return 0;
306
307	file = mock_file(i915);
308	if (IS_ERR(ptr: file))
309	return PTR_ERR(ptr: file);
310
311	ctx = live_context(i915, file);
312	if (IS_ERR(ptr: ctx)) {
313	err = PTR_ERR(ptr: ctx);
314	goto out_file;
315	}
316
317	engines = i915_gem_context_lock_engines(ctx);
318	count = engines->num_engines;
319
320	data = kcalloc(count, sizeof(*data), GFP_KERNEL);
321	if (!data) {
322	i915_gem_context_unlock_engines(ctx);
323	err = -ENOMEM;
324	goto out_file;
325	}
326
327	m = 0; /* Use the first context as our template for the engines */
328	for_each_gem_engine(ce, engines, it) {
329	err = intel_context_pin(ce);
330	if (err) {
331	i915_gem_context_unlock_engines(ctx);
332	goto out;
333	}
334	data[m++].ce[0] = intel_context_get(ce);
335	}
336	i915_gem_context_unlock_engines(ctx);
337
338	/* Clone the same set of engines into the other contexts */
339	for (n = 1; n < ARRAY_SIZE(data->ce); n++) {
340	ctx = live_context(i915, file);
341	if (IS_ERR(ptr: ctx)) {
342	err = PTR_ERR(ptr: ctx);
343	goto out;
344	}
345
346	for (m = 0; m < count; m++) {
347	if (!data[m].ce[0])
348	continue;
349
350	ce = intel_context_create(engine: data[m].ce[0]->engine);
351	if (IS_ERR(ptr: ce)) {
352	err = PTR_ERR(ptr: ce);
353	goto out;
354	}
355
356	err = intel_context_pin(ce);
357	if (err) {
358	intel_context_put(ce);
359	goto out;
360	}
361
362	data[m].ce[n] = ce;
363	}
364	}
365
366	for (n = 0; n < count; n++) {
367	struct kthread_worker *worker;
368
369	if (!data[n].ce[0])
370	continue;
371
372	worker = kthread_run_worker(0, "igt/parallel:%s",
373	data[n].ce[0]->engine->name);
374	if (IS_ERR(ptr: worker)) {
375	err = PTR_ERR(ptr: worker);
376	goto out;
377	}
378
379	data[n].worker = worker;
380	}
381
382	for (fn = func; !err && *fn; fn++) {
383	struct igt_live_test t;
384
385	err = igt_live_test_begin(t: &t, i915, func: __func__, name: "");
386	if (err)
387	break;
388
389	for (n = 0; n < count; n++) {
390	if (!data[n].ce[0])
391	continue;
392
393	data[n].result = 0;
394	kthread_init_work(&data[n].work, *fn);
395	kthread_queue_work(worker: data[n].worker, work: &data[n].work);
396	}
397
398	for (n = 0; n < count; n++) {
399	if (data[n].ce[0]) {
400	kthread_flush_work(work: &data[n].work);
401	if (data[n].result && !err)
402	err = data[n].result;
403	}
404	}
405
406	if (igt_live_test_end(t: &t)) {
407	err = err ?: -EIO;
408	break;
409	}
410	}
411
412	out:
413	for (n = 0; n < count; n++) {
414	for (m = 0; m < ARRAY_SIZE(data->ce); m++) {
415	if (!data[n].ce[m])
416	continue;
417
418	intel_context_unpin(ce: data[n].ce[m]);
419	intel_context_put(ce: data[n].ce[m]);
420	}
421
422	if (data[n].worker)
423	kthread_destroy_worker(worker: data[n].worker);
424	}
425	kfree(objp: data);
426	out_file:
427	fput(file);
428	return err;
429	}
430
431	static unsigned long real_page_count(struct drm_i915_gem_object *obj)
432	{
433	return huge_gem_object_phys_size(obj) >> PAGE_SHIFT;
434	}
435
436	static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
437	{
438	return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
439	}
440
441	static int gpu_fill(struct intel_context *ce,
442	struct drm_i915_gem_object *obj,
443	unsigned int dw)
444	{
445	struct i915_vma *vma;
446	int err;
447
448	GEM_BUG_ON(obj->base.size > ce->vm->total);
449	GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
450
451	vma = i915_vma_instance(obj, vm: ce->vm, NULL);
452	if (IS_ERR(ptr: vma))
453	return PTR_ERR(ptr: vma);
454
455	err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_HIGH | PIN_USER);
456	if (err)
457	return err;
458
459	/*
460	* Within the GTT the huge objects maps every page onto
461	* its 1024 real pages (using phys_pfn = dma_pfn % 1024).
462	* We set the nth dword within the page using the nth
463	* mapping via the GTT - this should exercise the GTT mapping
464	* whilst checking that each context provides a unique view
465	* into the object.
466	*/
467	err = igt_gpu_fill_dw(ce, vma,
468	offset: (dw * real_page_count(obj)) << PAGE_SHIFT |
469	(dw * sizeof(u32)),
470	count: real_page_count(obj),
471	val: dw);
472	i915_vma_unpin(vma);
473
474	return err;
475	}
476
477	static int cpu_fill(struct drm_i915_gem_object *obj, u32 value)
478	{
479	const bool has_llc = HAS_LLC(to_i915(obj->base.dev));
480	unsigned int need_flush;
481	unsigned long n, m;
482	int err;
483
484	i915_gem_object_lock(obj, NULL);
485	err = i915_gem_object_prepare_write(obj, needs_clflush: &need_flush);
486	if (err)
487	goto out;
488
489	for (n = 0; n < real_page_count(obj); n++) {
490	u32 *map;
491
492	map = kmap_local_page(i915_gem_object_get_page(obj, n));
493	for (m = 0; m < DW_PER_PAGE; m++)
494	map[m] = value;
495	if (!has_llc)
496	drm_clflush_virt_range(addr: map, PAGE_SIZE);
497	kunmap_local(map);
498	}
499
500	i915_gem_object_finish_access(obj);
501	obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU;
502	obj->write_domain = 0;
503	out:
504	i915_gem_object_unlock(obj);
505	return err;
506	}
507
508	static noinline int cpu_check(struct drm_i915_gem_object *obj,
509	unsigned int idx, unsigned int max)
510	{
511	unsigned int needs_flush;
512	unsigned long n;
513	int err;
514
515	i915_gem_object_lock(obj, NULL);
516	err = i915_gem_object_prepare_read(obj, needs_clflush: &needs_flush);
517	if (err)
518	goto out_unlock;
519
520	for (n = 0; n < real_page_count(obj); n++) {
521	u32 *map, m;
522
523	map = kmap_local_page(i915_gem_object_get_page(obj, n));
524	if (needs_flush & CLFLUSH_BEFORE)
525	drm_clflush_virt_range(addr: map, PAGE_SIZE);
526
527	for (m = 0; m < max; m++) {
528	if (map[m] != m) {
529	pr_err("%pS: Invalid value at object %d page %ld/%ld, offset %d/%d: found %x expected %x\n",
530	__builtin_return_address(0), idx,
531	n, real_page_count(obj), m, max,
532	map[m], m);
533	err = -EINVAL;
534	goto out_unmap;
535	}
536	}
537
538	for (; m < DW_PER_PAGE; m++) {
539	if (map[m] != STACK_MAGIC) {
540	pr_err("%pS: Invalid value at object %d page %ld, offset %d: found %x expected %x (uninitialised)\n",
541	__builtin_return_address(0), idx, n, m,
542	map[m], STACK_MAGIC);
543	err = -EINVAL;
544	goto out_unmap;
545	}
546	}
547
548	out_unmap:
549	kunmap_local(map);
550	if (err)
551	break;
552	}
553
554	i915_gem_object_finish_access(obj);
555	out_unlock:
556	i915_gem_object_unlock(obj);
557	return err;
558	}
559
560	static int file_add_object(struct file *file, struct drm_i915_gem_object *obj)
561	{
562	int err;
563
564	GEM_BUG_ON(obj->base.handle_count);
565
566	/* tie the object to the drm_file for easy reaping */
567	err = idr_alloc(&to_drm_file(f: file)->object_idr,
568	ptr: &obj->base, start: 1, end: 0, GFP_KERNEL);
569	if (err < 0)
570	return err;
571
572	i915_gem_object_get(obj);
573	obj->base.handle_count++;
574	return 0;
575	}
576
577	static struct drm_i915_gem_object *
578	create_test_object(struct i915_address_space *vm,
579	struct file *file,
580	struct list_head *objects)
581	{
582	struct drm_i915_gem_object *obj;
583	u64 size;
584	int err;
585
586	/* Keep in GEM's good graces */
587	intel_gt_retire_requests(gt: vm->gt);
588
589	size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
590	size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
591
592	obj = huge_gem_object(i915: vm->i915, DW_PER_PAGE * PAGE_SIZE, dma_size: size);
593	if (IS_ERR(ptr: obj))
594	return obj;
595
596	err = file_add_object(file, obj);
597	i915_gem_object_put(obj);
598	if (err)
599	return ERR_PTR(error: err);
600
601	err = cpu_fill(obj, STACK_MAGIC);
602	if (err) {
603	pr_err("Failed to fill object with cpu, err=%d\n",
604	err);
605	return ERR_PTR(error: err);
606	}
607
608	list_add_tail(new: &obj->st_link, head: objects);
609	return obj;
610	}
611
612	static unsigned long max_dwords(struct drm_i915_gem_object *obj)
613	{
614	unsigned long npages = fake_page_count(obj);
615
616	GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE));
617	return npages / DW_PER_PAGE;
618	}
619
620	static void throttle_release(struct i915_request **q, int count)
621	{
622	int i;
623
624	for (i = 0; i < count; i++) {
625	if (IS_ERR_OR_NULL(ptr: q[i]))
626	continue;
627
628	i915_request_put(fetch_and_zero(&q[i]));
629	}
630	}
631
632	static int throttle(struct intel_context *ce,
633	struct i915_request **q, int count)
634	{
635	int i;
636
637	if (!IS_ERR_OR_NULL(ptr: q[0])) {
638	if (i915_request_wait(rq: q[0],
639	I915_WAIT_INTERRUPTIBLE,
640	MAX_SCHEDULE_TIMEOUT) < 0)
641	return -EINTR;
642
643	i915_request_put(rq: q[0]);
644	}
645
646	for (i = 0; i < count - 1; i++)
647	q[i] = q[i + 1];
648
649	q[i] = intel_context_create_request(ce);
650	if (IS_ERR(ptr: q[i]))
651	return PTR_ERR(ptr: q[i]);
652
653	i915_request_get(rq: q[i]);
654	i915_request_add(rq: q[i]);
655
656	return 0;
657	}
658
659	static int igt_ctx_exec(void *arg)
660	{
661	struct drm_i915_private *i915 = arg;
662	struct intel_engine_cs *engine;
663	int err = -ENODEV;
664
665	/*
666	* Create a few different contexts (with different mm) and write
667	* through each ctx/mm using the GPU making sure those writes end
668	* up in the expected pages of our obj.
669	*/
670
671	if (!DRIVER_CAPS(i915)->has_logical_contexts)
672	return 0;
673
674	for_each_uabi_engine(engine, i915) {
675	struct drm_i915_gem_object *obj = NULL;
676	unsigned long ncontexts, ndwords, dw;
677	struct i915_request *tq[5] = {};
678	struct igt_live_test t;
679	IGT_TIMEOUT(end_time);
680	LIST_HEAD(objects);
681	struct file *file;
682
683	if (!intel_engine_can_store_dword(engine))
684	continue;
685
686	if (!engine->context_size)
687	continue; /* No logical context support in HW */
688
689	file = mock_file(i915);
690	if (IS_ERR(ptr: file))
691	return PTR_ERR(ptr: file);
692
693	err = igt_live_test_begin(t: &t, i915, func: __func__, name: engine->name);
694	if (err)
695	goto out_file;
696
697	ncontexts = 0;
698	ndwords = 0;
699	dw = 0;
700	while (!time_after(jiffies, end_time)) {
701	struct i915_gem_context *ctx;
702	struct intel_context *ce;
703
704	ctx = kernel_context(i915, NULL);
705	if (IS_ERR(ptr: ctx)) {
706	err = PTR_ERR(ptr: ctx);
707	goto out_file;
708	}
709
710	ce = i915_gem_context_get_engine(ctx, idx: engine->legacy_idx);
711	GEM_BUG_ON(IS_ERR(ce));
712
713	if (!obj) {
714	obj = create_test_object(vm: ce->vm, file, objects: &objects);
715	if (IS_ERR(ptr: obj)) {
716	err = PTR_ERR(ptr: obj);
717	intel_context_put(ce);
718	kernel_context_close(ctx);
719	goto out_file;
720	}
721	}
722
723	err = gpu_fill(ce, obj, dw);
724	if (err) {
725	pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
726	ndwords, dw, max_dwords(obj),
727	engine->name,
728	str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
729	err);
730	intel_context_put(ce);
731	kernel_context_close(ctx);
732	goto out_file;
733	}
734
735	err = throttle(ce, q: tq, ARRAY_SIZE(tq));
736	if (err) {
737	intel_context_put(ce);
738	kernel_context_close(ctx);
739	goto out_file;
740	}
741
742	if (++dw == max_dwords(obj)) {
743	obj = NULL;
744	dw = 0;
745	}
746
747	ndwords++;
748	ncontexts++;
749
750	intel_context_put(ce);
751	kernel_context_close(ctx);
752	}
753
754	pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
755	ncontexts, engine->name, ndwords);
756
757	ncontexts = dw = 0;
758	list_for_each_entry(obj, &objects, st_link) {
759	unsigned int rem =
760	min_t(unsigned int, ndwords - dw, max_dwords(obj));
761
762	err = cpu_check(obj, idx: ncontexts++, max: rem);
763	if (err)
764	break;
765
766	dw += rem;
767	}
768
769	out_file:
770	throttle_release(q: tq, ARRAY_SIZE(tq));
771	if (igt_live_test_end(t: &t))
772	err = -EIO;
773
774	fput(file);
775	if (err)
776	return err;
777
778	i915_gem_drain_freed_objects(i915);
779	}
780
781	return 0;
782	}
783
784	static int igt_shared_ctx_exec(void *arg)
785	{
786	struct drm_i915_private *i915 = arg;
787	struct i915_request *tq[5] = {};
788	struct i915_gem_context *parent;
789	struct intel_engine_cs *engine;
790	struct igt_live_test t;
791	struct file *file;
792	int err = 0;
793
794	/*
795	* Create a few different contexts with the same mm and write
796	* through each ctx using the GPU making sure those writes end
797	* up in the expected pages of our obj.
798	*/
799	if (!DRIVER_CAPS(i915)->has_logical_contexts)
800	return 0;
801
802	file = mock_file(i915);
803	if (IS_ERR(ptr: file))
804	return PTR_ERR(ptr: file);
805
806	parent = live_context(i915, file);
807	if (IS_ERR(ptr: parent)) {
808	err = PTR_ERR(ptr: parent);
809	goto out_file;
810	}
811
812	if (!parent->vm) { /* not full-ppgtt; nothing to share */
813	err = 0;
814	goto out_file;
815	}
816
817	err = igt_live_test_begin(t: &t, i915, func: __func__, name: "");
818	if (err)
819	goto out_file;
820
821	for_each_uabi_engine(engine, i915) {
822	unsigned long ncontexts, ndwords, dw;
823	struct drm_i915_gem_object *obj = NULL;
824	IGT_TIMEOUT(end_time);
825	LIST_HEAD(objects);
826
827	if (!intel_engine_can_store_dword(engine))
828	continue;
829
830	dw = 0;
831	ndwords = 0;
832	ncontexts = 0;
833	while (!time_after(jiffies, end_time)) {
834	struct i915_gem_context *ctx;
835	struct intel_context *ce;
836
837	ctx = kernel_context(i915, vm: parent->vm);
838	if (IS_ERR(ptr: ctx)) {
839	err = PTR_ERR(ptr: ctx);
840	goto out_test;
841	}
842
843	ce = i915_gem_context_get_engine(ctx, idx: engine->legacy_idx);
844	GEM_BUG_ON(IS_ERR(ce));
845
846	if (!obj) {
847	obj = create_test_object(vm: parent->vm,
848	file, objects: &objects);
849	if (IS_ERR(ptr: obj)) {
850	err = PTR_ERR(ptr: obj);
851	intel_context_put(ce);
852	kernel_context_close(ctx);
853	goto out_test;
854	}
855	}
856
857	err = gpu_fill(ce, obj, dw);
858	if (err) {
859	pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
860	ndwords, dw, max_dwords(obj),
861	engine->name,
862	str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
863	err);
864	intel_context_put(ce);
865	kernel_context_close(ctx);
866	goto out_test;
867	}
868
869	err = throttle(ce, q: tq, ARRAY_SIZE(tq));
870	if (err) {
871	intel_context_put(ce);
872	kernel_context_close(ctx);
873	goto out_test;
874	}
875
876	if (++dw == max_dwords(obj)) {
877	obj = NULL;
878	dw = 0;
879	}
880
881	ndwords++;
882	ncontexts++;
883
884	intel_context_put(ce);
885	kernel_context_close(ctx);
886	}
887	pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
888	ncontexts, engine->name, ndwords);
889
890	ncontexts = dw = 0;
891	list_for_each_entry(obj, &objects, st_link) {
892	unsigned int rem =
893	min_t(unsigned int, ndwords - dw, max_dwords(obj));
894
895	err = cpu_check(obj, idx: ncontexts++, max: rem);
896	if (err)
897	goto out_test;
898
899	dw += rem;
900	}
901
902	i915_gem_drain_freed_objects(i915);
903	}
904	out_test:
905	throttle_release(q: tq, ARRAY_SIZE(tq));
906	if (igt_live_test_end(t: &t))
907	err = -EIO;
908	out_file:
909	fput(file);
910	return err;
911	}
912
913	static int rpcs_query_batch(struct drm_i915_gem_object *rpcs,
914	struct i915_vma *vma,
915	struct intel_engine_cs *engine)
916	{
917	u32 *cmd;
918
919	GEM_BUG_ON(GRAPHICS_VER(vma->vm->i915) < 8);
920
921	cmd = i915_gem_object_pin_map(obj: rpcs, type: I915_MAP_WB);
922	if (IS_ERR(ptr: cmd))
923	return PTR_ERR(ptr: cmd);
924
925	*cmd++ = MI_STORE_REGISTER_MEM_GEN8;
926	*cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE(engine->mmio_base));
927	*cmd++ = lower_32_bits(i915_vma_offset(vma));
928	*cmd++ = upper_32_bits(i915_vma_offset(vma));
929	*cmd = MI_BATCH_BUFFER_END;
930
931	__i915_gem_object_flush_map(obj: rpcs, offset: 0, size: 64);
932	i915_gem_object_unpin_map(obj: rpcs);
933
934	intel_gt_chipset_flush(gt: vma->vm->gt);
935
936	return 0;
937	}
938
939	static int
940	emit_rpcs_query(struct drm_i915_gem_object *obj,
941	struct intel_context *ce,
942	struct i915_request **rq_out)
943	{
944	struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
945	struct i915_request *rq;
946	struct i915_gem_ww_ctx ww;
947	struct i915_vma *batch;
948	struct i915_vma *vma;
949	struct drm_i915_gem_object *rpcs;
950	int err;
951
952	GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
953
954	if (GRAPHICS_VER(i915) < 8)
955	return -EINVAL;
956
957	vma = i915_vma_instance(obj, vm: ce->vm, NULL);
958	if (IS_ERR(ptr: vma))
959	return PTR_ERR(ptr: vma);
960
961	rpcs = i915_gem_object_create_internal(i915, PAGE_SIZE);
962	if (IS_ERR(ptr: rpcs))
963	return PTR_ERR(ptr: rpcs);
964
965	i915_gem_ww_ctx_init(ctx: &ww, intr: false);
966
967	batch = i915_vma_instance(obj: rpcs, vm: ce->vm, NULL);
968	if (IS_ERR(ptr: batch)) {
969	err = PTR_ERR(ptr: batch);
970	goto err_put;
971	}
972
973	retry:
974	err = i915_gem_object_lock(obj, ww: &ww);
975	if (!err)
976	err = i915_gem_object_lock(obj: rpcs, ww: &ww);
977	if (!err)
978	err = i915_gem_object_set_to_gtt_domain(obj, write: false);
979	if (!err)
980	err = i915_vma_pin_ww(vma, ww: &ww, size: 0, alignment: 0, PIN_USER);
981	if (err)
982	goto err_put;
983
984	err = i915_vma_pin_ww(vma: batch, ww: &ww, size: 0, alignment: 0, PIN_USER);
985	if (err)
986	goto err_vma;
987
988	err = rpcs_query_batch(rpcs, vma, engine: ce->engine);
989	if (err)
990	goto err_batch;
991
992	rq = i915_request_create(ce);
993	if (IS_ERR(ptr: rq)) {
994	err = PTR_ERR(ptr: rq);
995	goto err_batch;
996	}
997
998	err = i915_vma_move_to_active(vma: batch, rq, flags: 0);
999	if (err)
1000	goto skip_request;
1001
1002	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1003	if (err)
1004	goto skip_request;
1005
1006	if (rq->engine->emit_init_breadcrumb) {
1007	err = rq->engine->emit_init_breadcrumb(rq);
1008	if (err)
1009	goto skip_request;
1010	}
1011
1012	err = rq->engine->emit_bb_start(rq,
1013	i915_vma_offset(vma: batch),
1014	i915_vma_size(vma: batch),
1015	0);
1016	if (err)
1017	goto skip_request;
1018
1019	*rq_out = i915_request_get(rq);
1020
1021	skip_request:
1022	if (err)
1023	i915_request_set_error_once(rq, error: err);
1024	i915_request_add(rq);
1025	err_batch:
1026	i915_vma_unpin(vma: batch);
1027	err_vma:
1028	i915_vma_unpin(vma);
1029	err_put:
1030	if (err == -EDEADLK) {
1031	err = i915_gem_ww_ctx_backoff(ctx: &ww);
1032	if (!err)
1033	goto retry;
1034	}
1035	i915_gem_ww_ctx_fini(ctx: &ww);
1036	i915_gem_object_put(obj: rpcs);
1037	return err;
1038	}
1039
1040	#define TEST_IDLE BIT(0)
1041	#define TEST_BUSY BIT(1)
1042	#define TEST_RESET BIT(2)
1043
1044	static int
1045	__sseu_prepare(const char *name,
1046	unsigned int flags,
1047	struct intel_context *ce,
1048	struct igt_spinner **spin)
1049	{
1050	struct i915_request *rq;
1051	int ret;
1052
1053	*spin = NULL;
1054	if (!(flags & (TEST_BUSY | TEST_RESET)))
1055	return 0;
1056
1057	*spin = kzalloc(sizeof(**spin), GFP_KERNEL);
1058	if (!*spin)
1059	return -ENOMEM;
1060
1061	ret = igt_spinner_init(spin: *spin, gt: ce->engine->gt);
1062	if (ret)
1063	goto err_free;
1064
1065	rq = igt_spinner_create_request(spin: *spin, ce, MI_NOOP);
1066	if (IS_ERR(ptr: rq)) {
1067	ret = PTR_ERR(ptr: rq);
1068	goto err_fini;
1069	}
1070
1071	i915_request_add(rq);
1072
1073	if (!igt_wait_for_spinner(spin: *spin, rq)) {
1074	pr_err("%s: Spinner failed to start!\n", name);
1075	ret = -ETIMEDOUT;
1076	goto err_end;
1077	}
1078
1079	return 0;
1080
1081	err_end:
1082	igt_spinner_end(spin: *spin);
1083	err_fini:
1084	igt_spinner_fini(spin: *spin);
1085	err_free:
1086	kfree(fetch_and_zero(spin));
1087	return ret;
1088	}
1089
1090	static int
1091	__read_slice_count(struct intel_context *ce,
1092	struct drm_i915_gem_object *obj,
1093	struct igt_spinner *spin,
1094	u32 *rpcs)
1095	{
1096	struct i915_request *rq = NULL;
1097	u32 s_mask, s_shift;
1098	unsigned int cnt;
1099	u32 *buf, val;
1100	long ret;
1101
1102	ret = emit_rpcs_query(obj, ce, rq_out: &rq);
1103	if (ret)
1104	return ret;
1105
1106	if (spin)
1107	igt_spinner_end(spin);
1108
1109	ret = i915_request_wait(rq, flags: 0, MAX_SCHEDULE_TIMEOUT);
1110	i915_request_put(rq);
1111	if (ret < 0)
1112	return ret;
1113
1114	buf = i915_gem_object_pin_map_unlocked(obj, type: I915_MAP_WB);
1115	if (IS_ERR(ptr: buf)) {
1116	ret = PTR_ERR(ptr: buf);
1117	return ret;
1118	}
1119
1120	if (GRAPHICS_VER(ce->engine->i915) >= 11) {
1121	s_mask = GEN11_RPCS_S_CNT_MASK;
1122	s_shift = GEN11_RPCS_S_CNT_SHIFT;
1123	} else {
1124	s_mask = GEN8_RPCS_S_CNT_MASK;
1125	s_shift = GEN8_RPCS_S_CNT_SHIFT;
1126	}
1127
1128	val = *buf;
1129	cnt = (val & s_mask) >> s_shift;
1130	*rpcs = val;
1131
1132	i915_gem_object_unpin_map(obj);
1133
1134	return cnt;
1135	}
1136
1137	static int
1138	__check_rpcs(const char *name, u32 rpcs, int slices, unsigned int expected,
1139	const char *prefix, const char *suffix)
1140	{
1141	if (slices == expected)
1142	return 0;
1143
1144	if (slices < 0) {
1145	pr_err("%s: %s read slice count failed with %d%s\n",
1146	name, prefix, slices, suffix);
1147	return slices;
1148	}
1149
1150	pr_err("%s: %s slice count %d is not %u%s\n",
1151	name, prefix, slices, expected, suffix);
1152
1153	pr_info("RPCS=0x%x; %u%sx%u%s\n",
1154	rpcs, slices,
1155	(rpcs & GEN8_RPCS_S_CNT_ENABLE) ? "*" : "",
1156	(rpcs & GEN8_RPCS_SS_CNT_MASK) >> GEN8_RPCS_SS_CNT_SHIFT,
1157	(rpcs & GEN8_RPCS_SS_CNT_ENABLE) ? "*" : "");
1158
1159	return -EINVAL;
1160	}
1161
1162	static int
1163	__sseu_finish(const char *name,
1164	unsigned int flags,
1165	struct intel_context *ce,
1166	struct drm_i915_gem_object *obj,
1167	unsigned int expected,
1168	struct igt_spinner *spin)
1169	{
1170	unsigned int slices = hweight32(ce->engine->sseu.slice_mask);
1171	u32 rpcs = 0;
1172	int ret = 0;
1173
1174	if (flags & TEST_RESET) {
1175	ret = intel_engine_reset(engine: ce->engine, reason: "sseu");
1176	if (ret)
1177	goto out;
1178	}
1179
1180	ret = __read_slice_count(ce, obj,
1181	spin: flags & TEST_RESET ? NULL : spin, rpcs: &rpcs);
1182	ret = __check_rpcs(name, rpcs, slices: ret, expected, prefix: "Context", suffix: "!");
1183	if (ret)
1184	goto out;
1185
1186	ret = __read_slice_count(ce: ce->engine->kernel_context, obj, NULL, rpcs: &rpcs);
1187	ret = __check_rpcs(name, rpcs, slices: ret, expected: slices, prefix: "Kernel context", suffix: "!");
1188
1189	out:
1190	if (spin)
1191	igt_spinner_end(spin);
1192
1193	if ((flags & TEST_IDLE) && ret == 0) {
1194	ret = igt_flush_test(i915: ce->engine->i915);
1195	if (ret)
1196	return ret;
1197
1198	ret = __read_slice_count(ce, obj, NULL, rpcs: &rpcs);
1199	ret = __check_rpcs(name, rpcs, slices: ret, expected,
1200	prefix: "Context", suffix: " after idle!");
1201	}
1202
1203	return ret;
1204	}
1205
1206	static int
1207	__sseu_test(const char *name,
1208	unsigned int flags,
1209	struct intel_context *ce,
1210	struct drm_i915_gem_object *obj,
1211	struct intel_sseu sseu)
1212	{
1213	struct igt_spinner *spin = NULL;
1214	int ret;
1215
1216	intel_engine_pm_get(engine: ce->engine);
1217
1218	ret = __sseu_prepare(name, flags, ce, spin: &spin);
1219	if (ret)
1220	goto out_pm;
1221
1222	ret = intel_context_reconfigure_sseu(ce, sseu);
1223	if (ret)
1224	goto out_spin;
1225
1226	ret = __sseu_finish(name, flags, ce, obj,
1227	hweight32(sseu.slice_mask), spin);
1228
1229	out_spin:
1230	if (spin) {
1231	igt_spinner_end(spin);
1232	igt_spinner_fini(spin);
1233	kfree(objp: spin);
1234	}
1235	out_pm:
1236	intel_engine_pm_put(engine: ce->engine);
1237	return ret;
1238	}
1239
1240	static int
1241	__igt_ctx_sseu(struct drm_i915_private *i915,
1242	const char *name,
1243	unsigned int flags)
1244	{
1245	struct drm_i915_gem_object *obj;
1246	int inst = 0;
1247	int ret = 0;
1248
1249	if (GRAPHICS_VER(i915) < 9)
1250	return 0;
1251
1252	if (flags & TEST_RESET)
1253	igt_global_reset_lock(gt: to_gt(i915));
1254
1255	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1256	if (IS_ERR(ptr: obj)) {
1257	ret = PTR_ERR(ptr: obj);
1258	goto out_unlock;
1259	}
1260
1261	do {
1262	struct intel_engine_cs *engine;
1263	struct intel_context *ce;
1264	struct intel_sseu pg_sseu;
1265
1266	engine = intel_engine_lookup_user(i915,
1267	class: I915_ENGINE_CLASS_RENDER,
1268	instance: inst++);
1269	if (!engine)
1270	break;
1271
1272	if (hweight32(engine->sseu.slice_mask) < 2)
1273	continue;
1274
1275	if (!engine->gt->info.sseu.has_slice_pg)
1276	continue;
1277
1278	/*
1279	* Gen11 VME friendly power-gated configuration with
1280	* half enabled sub-slices.
1281	*/
1282	pg_sseu = engine->sseu;
1283	pg_sseu.slice_mask = 1;
1284	pg_sseu.subslice_mask =
1285	~(~0 << (hweight32(engine->sseu.subslice_mask) / 2));
1286
1287	pr_info("%s: SSEU subtest '%s', flags=%x, def_slices=%u, pg_slices=%u\n",
1288	engine->name, name, flags,
1289	hweight32(engine->sseu.slice_mask),
1290	hweight32(pg_sseu.slice_mask));
1291
1292	ce = intel_context_create(engine);
1293	if (IS_ERR(ptr: ce)) {
1294	ret = PTR_ERR(ptr: ce);
1295	goto out_put;
1296	}
1297
1298	ret = intel_context_pin(ce);
1299	if (ret)
1300	goto out_ce;
1301
1302	/* First set the default mask. */
1303	ret = __sseu_test(name, flags, ce, obj, sseu: engine->sseu);
1304	if (ret)
1305	goto out_unpin;
1306
1307	/* Then set a power-gated configuration. */
1308	ret = __sseu_test(name, flags, ce, obj, sseu: pg_sseu);
1309	if (ret)
1310	goto out_unpin;
1311
1312	/* Back to defaults. */
1313	ret = __sseu_test(name, flags, ce, obj, sseu: engine->sseu);
1314	if (ret)
1315	goto out_unpin;
1316
1317	/* One last power-gated configuration for the road. */
1318	ret = __sseu_test(name, flags, ce, obj, sseu: pg_sseu);
1319	if (ret)
1320	goto out_unpin;
1321
1322	out_unpin:
1323	intel_context_unpin(ce);
1324	out_ce:
1325	intel_context_put(ce);
1326	} while (!ret);
1327
1328	if (igt_flush_test(i915))
1329	ret = -EIO;
1330
1331	out_put:
1332	i915_gem_object_put(obj);
1333
1334	out_unlock:
1335	if (flags & TEST_RESET)
1336	igt_global_reset_unlock(gt: to_gt(i915));
1337
1338	if (ret)
1339	pr_err("%s: Failed with %d!\n", name, ret);
1340
1341	return ret;
1342	}
1343
1344	static int igt_ctx_sseu(void *arg)
1345	{
1346	struct {
1347	const char *name;
1348	unsigned int flags;
1349	} *phase, phases[] = {
1350	{ .name = "basic", .flags = 0 },
1351	{ .name = "idle", .flags = TEST_IDLE },
1352	{ .name = "busy", .flags = TEST_BUSY },
1353	{ .name = "busy-reset", .flags = TEST_BUSY | TEST_RESET },
1354	{ .name = "busy-idle", .flags = TEST_BUSY | TEST_IDLE },
1355	{ .name = "reset-idle", .flags = TEST_RESET | TEST_IDLE },
1356	};
1357	unsigned int i;
1358	int ret = 0;
1359
1360	for (i = 0, phase = phases; ret == 0 && i < ARRAY_SIZE(phases);
1361	i++, phase++)
1362	ret = __igt_ctx_sseu(i915: arg, name: phase->name, flags: phase->flags);
1363
1364	return ret;
1365	}
1366
1367	static int igt_ctx_readonly(void *arg)
1368	{
1369	struct drm_i915_private *i915 = arg;
1370	unsigned long idx, ndwords, dw, num_engines;
1371	struct drm_i915_gem_object *obj = NULL;
1372	struct i915_request *tq[5] = {};
1373	struct i915_gem_engines_iter it;
1374	struct i915_address_space *vm;
1375	struct i915_gem_context *ctx;
1376	struct intel_context *ce;
1377	struct igt_live_test t;
1378	I915_RND_STATE(prng);
1379	IGT_TIMEOUT(end_time);
1380	LIST_HEAD(objects);
1381	struct file *file;
1382	int err = -ENODEV;
1383
1384	/*
1385	* Create a few read-only objects (with the occasional writable object)
1386	* and try to write into these object checking that the GPU discards
1387	* any write to a read-only object.
1388	*/
1389
1390	file = mock_file(i915);
1391	if (IS_ERR(ptr: file))
1392	return PTR_ERR(ptr: file);
1393
1394	err = igt_live_test_begin(t: &t, i915, func: __func__, name: "");
1395	if (err)
1396	goto out_file;
1397
1398	ctx = live_context(i915, file);
1399	if (IS_ERR(ptr: ctx)) {
1400	err = PTR_ERR(ptr: ctx);
1401	goto out_file;
1402	}
1403
1404	vm = ctx->vm ?: &to_gt(i915)->ggtt->alias->vm;
1405	if (!vm || !vm->has_read_only) {
1406	err = 0;
1407	goto out_file;
1408	}
1409
1410	num_engines = 0;
1411	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it)
1412	if (intel_engine_can_store_dword(engine: ce->engine))
1413	num_engines++;
1414	i915_gem_context_unlock_engines(ctx);
1415
1416	ndwords = 0;
1417	dw = 0;
1418	while (!time_after(jiffies, end_time)) {
1419	for_each_gem_engine(ce,
1420	i915_gem_context_lock_engines(ctx), it) {
1421	if (!intel_engine_can_store_dword(engine: ce->engine))
1422	continue;
1423
1424	if (!obj) {
1425	obj = create_test_object(vm: ce->vm, file, objects: &objects);
1426	if (IS_ERR(ptr: obj)) {
1427	err = PTR_ERR(ptr: obj);
1428	i915_gem_context_unlock_engines(ctx);
1429	goto out_file;
1430	}
1431
1432	if (prandom_u32_state(state: &prng) & 1)
1433	i915_gem_object_set_readonly(obj);
1434	}
1435
1436	err = gpu_fill(ce, obj, dw);
1437	if (err) {
1438	pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
1439	ndwords, dw, max_dwords(obj),
1440	ce->engine->name,
1441	str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
1442	err);
1443	i915_gem_context_unlock_engines(ctx);
1444	goto out_file;
1445	}
1446
1447	err = throttle(ce, q: tq, ARRAY_SIZE(tq));
1448	if (err) {
1449	i915_gem_context_unlock_engines(ctx);
1450	goto out_file;
1451	}
1452
1453	if (++dw == max_dwords(obj)) {
1454	obj = NULL;
1455	dw = 0;
1456	}
1457	ndwords++;
1458	}
1459	i915_gem_context_unlock_engines(ctx);
1460	}
1461	pr_info("Submitted %lu dwords (across %lu engines)\n",
1462	ndwords, num_engines);
1463
1464	dw = 0;
1465	idx = 0;
1466	list_for_each_entry(obj, &objects, st_link) {
1467	unsigned int rem =
1468	min_t(unsigned int, ndwords - dw, max_dwords(obj));
1469	unsigned int num_writes;
1470
1471	num_writes = rem;
1472	if (i915_gem_object_is_readonly(obj))
1473	num_writes = 0;
1474
1475	err = cpu_check(obj, idx: idx++, max: num_writes);
1476	if (err)
1477	break;
1478
1479	dw += rem;
1480	}
1481
1482	out_file:
1483	throttle_release(q: tq, ARRAY_SIZE(tq));
1484	if (igt_live_test_end(t: &t))
1485	err = -EIO;
1486
1487	fput(file);
1488	return err;
1489	}
1490
1491	static int check_scratch(struct i915_address_space *vm, u64 offset)
1492	{
1493	struct drm_mm_node *node;
1494
1495	mutex_lock(&vm->mutex);
1496	node = __drm_mm_interval_first(mm: &vm->mm,
1497	start: offset, last: offset + sizeof(u32) - 1);
1498	mutex_unlock(lock: &vm->mutex);
1499	if (!node || node->start > offset)
1500	return 0;
1501
1502	GEM_BUG_ON(offset >= node->start + node->size);
1503
1504	pr_err("Target offset 0x%08x_%08x overlaps with a node in the mm!\n",
1505	upper_32_bits(offset), lower_32_bits(offset));
1506	return -EINVAL;
1507	}
1508
1509	static int write_to_scratch(struct i915_gem_context *ctx,
1510	struct intel_engine_cs *engine,
1511	struct drm_i915_gem_object *obj,
1512	u64 offset, u32 value)
1513	{
1514	struct drm_i915_private *i915 = ctx->i915;
1515	struct i915_address_space *vm;
1516	struct i915_request *rq;
1517	struct i915_vma *vma;
1518	u32 *cmd;
1519	int err;
1520
1521	GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1522
1523	err = check_scratch(vm: ctx->vm, offset);
1524	if (err)
1525	return err;
1526
1527	cmd = i915_gem_object_pin_map_unlocked(obj, type: I915_MAP_WB);
1528	if (IS_ERR(ptr: cmd))
1529	return PTR_ERR(ptr: cmd);
1530
1531	*cmd++ = MI_STORE_DWORD_IMM_GEN4;
1532	if (GRAPHICS_VER(i915) >= 8) {
1533	*cmd++ = lower_32_bits(offset);
1534	*cmd++ = upper_32_bits(offset);
1535	} else {
1536	*cmd++ = 0;
1537	*cmd++ = offset;
1538	}
1539	*cmd++ = value;
1540	*cmd = MI_BATCH_BUFFER_END;
1541	__i915_gem_object_flush_map(obj, offset: 0, size: 64);
1542	i915_gem_object_unpin_map(obj);
1543
1544	intel_gt_chipset_flush(gt: engine->gt);
1545
1546	vm = i915_gem_context_get_eb_vm(ctx);
1547	vma = i915_vma_instance(obj, vm, NULL);
1548	if (IS_ERR(ptr: vma)) {
1549	err = PTR_ERR(ptr: vma);
1550	goto out_vm;
1551	}
1552
1553	err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_USER | PIN_OFFSET_FIXED);
1554	if (err)
1555	goto out_vm;
1556
1557	rq = igt_request_alloc(ctx, engine);
1558	if (IS_ERR(ptr: rq)) {
1559	err = PTR_ERR(ptr: rq);
1560	goto err_unpin;
1561	}
1562
1563	err = igt_vma_move_to_active_unlocked(vma, rq, flags: 0);
1564	if (err)
1565	goto skip_request;
1566
1567	if (rq->engine->emit_init_breadcrumb) {
1568	err = rq->engine->emit_init_breadcrumb(rq);
1569	if (err)
1570	goto skip_request;
1571	}
1572
1573	err = engine->emit_bb_start(rq, i915_vma_offset(vma),
1574	i915_vma_size(vma), 0);
1575	if (err)
1576	goto skip_request;
1577
1578	i915_vma_unpin(vma);
1579
1580	i915_request_add(rq);
1581
1582	goto out_vm;
1583	skip_request:
1584	i915_request_set_error_once(rq, error: err);
1585	i915_request_add(rq);
1586	err_unpin:
1587	i915_vma_unpin(vma);
1588	out_vm:
1589	i915_vm_put(vm);
1590
1591	if (!err)
1592	err = i915_gem_object_wait(obj, flags: 0, MAX_SCHEDULE_TIMEOUT);
1593
1594	return err;
1595	}
1596
1597	static int read_from_scratch(struct i915_gem_context *ctx,
1598	struct intel_engine_cs *engine,
1599	struct drm_i915_gem_object *obj,
1600	u64 offset, u32 *value)
1601	{
1602	struct drm_i915_private *i915 = ctx->i915;
1603	struct i915_address_space *vm;
1604	const u32 result = 0x100;
1605	struct i915_request *rq;
1606	struct i915_vma *vma;
1607	unsigned int flags;
1608	u32 *cmd;
1609	int err;
1610
1611	GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1612
1613	err = check_scratch(vm: ctx->vm, offset);
1614	if (err)
1615	return err;
1616
1617	if (GRAPHICS_VER(i915) >= 8) {
1618	const u32 GPR0 = engine->mmio_base + 0x600;
1619
1620	vm = i915_gem_context_get_eb_vm(ctx);
1621	vma = i915_vma_instance(obj, vm, NULL);
1622	if (IS_ERR(ptr: vma)) {
1623	err = PTR_ERR(ptr: vma);
1624	goto out_vm;
1625	}
1626
1627	err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_USER | PIN_OFFSET_FIXED);
1628	if (err)
1629	goto out_vm;
1630
1631	cmd = i915_gem_object_pin_map_unlocked(obj, type: I915_MAP_WB);
1632	if (IS_ERR(ptr: cmd)) {
1633	err = PTR_ERR(ptr: cmd);
1634	goto err_unpin;
1635	}
1636
1637	memset(cmd, POISON_INUSE, PAGE_SIZE);
1638	*cmd++ = MI_LOAD_REGISTER_MEM_GEN8;
1639	*cmd++ = GPR0;
1640	*cmd++ = lower_32_bits(offset);
1641	*cmd++ = upper_32_bits(offset);
1642	*cmd++ = MI_STORE_REGISTER_MEM_GEN8;
1643	*cmd++ = GPR0;
1644	*cmd++ = result;
1645	*cmd++ = 0;
1646	*cmd = MI_BATCH_BUFFER_END;
1647
1648	i915_gem_object_flush_map(obj);
1649	i915_gem_object_unpin_map(obj);
1650
1651	flags = 0;
1652	} else {
1653	const u32 reg = engine->mmio_base + 0x420;
1654
1655	/* hsw: register access even to 3DPRIM! is protected */
1656	vm = i915_vm_get(vm: &engine->gt->ggtt->vm);
1657	vma = i915_vma_instance(obj, vm, NULL);
1658	if (IS_ERR(ptr: vma)) {
1659	err = PTR_ERR(ptr: vma);
1660	goto out_vm;
1661	}
1662
1663	err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_GLOBAL);
1664	if (err)
1665	goto out_vm;
1666
1667	cmd = i915_gem_object_pin_map_unlocked(obj, type: I915_MAP_WB);
1668	if (IS_ERR(ptr: cmd)) {
1669	err = PTR_ERR(ptr: cmd);
1670	goto err_unpin;
1671	}
1672
1673	memset(cmd, POISON_INUSE, PAGE_SIZE);
1674	*cmd++ = MI_LOAD_REGISTER_MEM;
1675	*cmd++ = reg;
1676	*cmd++ = offset;
1677	*cmd++ = MI_STORE_REGISTER_MEM | MI_USE_GGTT;
1678	*cmd++ = reg;
1679	*cmd++ = i915_vma_offset(vma) + result;
1680	*cmd = MI_BATCH_BUFFER_END;
1681
1682	i915_gem_object_flush_map(obj);
1683	i915_gem_object_unpin_map(obj);
1684
1685	flags = I915_DISPATCH_SECURE;
1686	}
1687
1688	intel_gt_chipset_flush(gt: engine->gt);
1689
1690	rq = igt_request_alloc(ctx, engine);
1691	if (IS_ERR(ptr: rq)) {
1692	err = PTR_ERR(ptr: rq);
1693	goto err_unpin;
1694	}
1695
1696	err = igt_vma_move_to_active_unlocked(vma, rq, EXEC_OBJECT_WRITE);
1697	if (err)
1698	goto skip_request;
1699
1700	if (rq->engine->emit_init_breadcrumb) {
1701	err = rq->engine->emit_init_breadcrumb(rq);
1702	if (err)
1703	goto skip_request;
1704	}
1705
1706	err = engine->emit_bb_start(rq, i915_vma_offset(vma),
1707	i915_vma_size(vma), flags);
1708	if (err)
1709	goto skip_request;
1710
1711	i915_vma_unpin(vma);
1712
1713	i915_request_add(rq);
1714
1715	i915_gem_object_lock(obj, NULL);
1716	err = i915_gem_object_set_to_cpu_domain(obj, write: false);
1717	i915_gem_object_unlock(obj);
1718	if (err)
1719	goto out_vm;
1720
1721	cmd = i915_gem_object_pin_map_unlocked(obj, type: I915_MAP_WB);
1722	if (IS_ERR(ptr: cmd)) {
1723	err = PTR_ERR(ptr: cmd);
1724	goto out_vm;
1725	}
1726
1727	*value = cmd[result / sizeof(*cmd)];
1728	i915_gem_object_unpin_map(obj);
1729
1730	goto out_vm;
1731	skip_request:
1732	i915_request_set_error_once(rq, error: err);
1733	i915_request_add(rq);
1734	err_unpin:
1735	i915_vma_unpin(vma);
1736	out_vm:
1737	i915_vm_put(vm);
1738
1739	if (!err)
1740	err = i915_gem_object_wait(obj, flags: 0, MAX_SCHEDULE_TIMEOUT);
1741
1742	return err;
1743	}
1744
1745	static int check_scratch_page(struct i915_gem_context *ctx, u32 *out)
1746	{
1747	struct i915_address_space *vm;
1748	u32 *vaddr;
1749	int err = 0;
1750
1751	vm = ctx->vm;
1752	if (!vm)
1753	return -ENODEV;
1754
1755	if (!vm->scratch[0]) {
1756	pr_err("No scratch page!\n");
1757	return -EINVAL;
1758	}
1759
1760	vaddr = __px_vaddr(p: vm->scratch[0]);
1761
1762	memcpy(out, vaddr, sizeof(*out));
1763	if (memchr_inv(p: vaddr, c: *out, PAGE_SIZE)) {
1764	pr_err("Inconsistent initial state of scratch page!\n");
1765	err = -EINVAL;
1766	}
1767
1768	return err;
1769	}
1770
1771	static int igt_vm_isolation(void *arg)
1772	{
1773	struct drm_i915_private *i915 = arg;
1774	struct i915_gem_context *ctx_a, *ctx_b;
1775	struct drm_i915_gem_object *obj_a, *obj_b;
1776	unsigned long num_engines, count;
1777	struct intel_engine_cs *engine;
1778	struct igt_live_test t;
1779	I915_RND_STATE(prng);
1780	struct file *file;
1781	u64 vm_total;
1782	u32 expected;
1783	int err;
1784
1785	if (GRAPHICS_VER(i915) < 7)
1786	return 0;
1787
1788	/*
1789	* The simple goal here is that a write into one context is not
1790	* observed in a second (separate page tables and scratch).
1791	*/
1792
1793	file = mock_file(i915);
1794	if (IS_ERR(ptr: file))
1795	return PTR_ERR(ptr: file);
1796
1797	err = igt_live_test_begin(t: &t, i915, func: __func__, name: "");
1798	if (err)
1799	goto out_file;
1800
1801	ctx_a = live_context(i915, file);
1802	if (IS_ERR(ptr: ctx_a)) {
1803	err = PTR_ERR(ptr: ctx_a);
1804	goto out_file;
1805	}
1806
1807	ctx_b = live_context(i915, file);
1808	if (IS_ERR(ptr: ctx_b)) {
1809	err = PTR_ERR(ptr: ctx_b);
1810	goto out_file;
1811	}
1812
1813	/* We can only test vm isolation, if the vm are distinct */
1814	if (ctx_a->vm == ctx_b->vm)
1815	goto out_file;
1816
1817	/* Read the initial state of the scratch page */
1818	err = check_scratch_page(ctx: ctx_a, out: &expected);
1819	if (err)
1820	goto out_file;
1821
1822	err = check_scratch_page(ctx: ctx_b, out: &expected);
1823	if (err)
1824	goto out_file;
1825
1826	vm_total = ctx_a->vm->total;
1827	GEM_BUG_ON(ctx_b->vm->total != vm_total);
1828
1829	obj_a = i915_gem_object_create_internal(i915, PAGE_SIZE);
1830	if (IS_ERR(ptr: obj_a)) {
1831	err = PTR_ERR(ptr: obj_a);
1832	goto out_file;
1833	}
1834
1835	obj_b = i915_gem_object_create_internal(i915, PAGE_SIZE);
1836	if (IS_ERR(ptr: obj_b)) {
1837	err = PTR_ERR(ptr: obj_b);
1838	goto put_a;
1839	}
1840
1841	count = 0;
1842	num_engines = 0;
1843	for_each_uabi_engine(engine, i915) {
1844	IGT_TIMEOUT(end_time);
1845	unsigned long this = 0;
1846
1847	if (!intel_engine_can_store_dword(engine))
1848	continue;
1849
1850	/* Not all engines have their own GPR! */
1851	if (GRAPHICS_VER(i915) < 8 && engine->class != RENDER_CLASS)
1852	continue;
1853
1854	while (!__igt_timeout(timeout: end_time, NULL)) {
1855	u32 value = 0xc5c5c5c5;
1856	u64 offset;
1857
1858	/* Leave enough space at offset 0 for the batch */
1859	offset = igt_random_offset(state: &prng,
1860	I915_GTT_PAGE_SIZE, end: vm_total,
1861	len: sizeof(u32), alignof_dword);
1862
1863	err = write_to_scratch(ctx: ctx_a, engine, obj: obj_a,
1864	offset, value: 0xdeadbeef);
1865	if (err == 0)
1866	err = read_from_scratch(ctx: ctx_b, engine, obj: obj_b,
1867	offset, value: &value);
1868	if (err)
1869	goto put_b;
1870
1871	if (value != expected) {
1872	pr_err("%s: Read %08x from scratch (offset 0x%08x_%08x), after %lu reads!\n",
1873	engine->name, value,
1874	upper_32_bits(offset),
1875	lower_32_bits(offset),
1876	this);
1877	err = -EINVAL;
1878	goto put_b;
1879	}
1880
1881	this++;
1882	}
1883	count += this;
1884	num_engines++;
1885	}
1886	pr_info("Checked %lu scratch offsets across %lu engines\n",
1887	count, num_engines);
1888
1889	put_b:
1890	i915_gem_object_put(obj: obj_b);
1891	put_a:
1892	i915_gem_object_put(obj: obj_a);
1893	out_file:
1894	if (igt_live_test_end(t: &t))
1895	err = -EIO;
1896	fput(file);
1897	return err;
1898	}
1899
1900	int i915_gem_context_live_selftests(struct drm_i915_private *i915)
1901	{
1902	static const struct i915_subtest tests[] = {
1903	SUBTEST(live_nop_switch),
1904	SUBTEST(live_parallel_switch),
1905	SUBTEST(igt_ctx_exec),
1906	SUBTEST(igt_ctx_readonly),
1907	SUBTEST(igt_ctx_sseu),
1908	SUBTEST(igt_shared_ctx_exec),
1909	SUBTEST(igt_vm_isolation),
1910	};
1911
1912	if (intel_gt_is_wedged(gt: to_gt(i915)))
1913	return 0;
1914
1915	return i915_live_subtests(tests, i915);
1916	}
1917