/* SPDX-License-Identifier: BSD-3-Clause

*

* Copyright(c) 2021 Intel Corporation. All rights reserved.

*

* Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>

*/

#ifndef __SOF_COHERENT_H__

#define __SOF_COHERENT_H__

#include <stdbool.h>

#include <stddef.h>

#include <stdint.h>

#include <rtos/spinlock.h>

#include <sof/list.h>

#include <sof/lib/memory.h>

#include <sof/lib/cpu.h>

#define __coherent __aligned(PLATFORM_DCACHE_ALIGN)

/*

* The coherent API allows optimized access to memory by multiple cores, using

* cache, taking care about coherence. The intended use is to let cores acquire

* ownership of such shared objects, use them, and then release them, possibly

* to be re-acquired by other cores. Such shared objects must only be accessed

* via this API. It's designed to be primarily used with dynamically allocated

* objects because of their well-defined life span. It can also be used with

* objects from .data or .bss sections but greater care must be takenwith them

* to strictly follow the API flow.

*

* The API assumes, that in the beginning no core has cache lines associated

* with the memory area, used with it. That is true for dynamically allocated

* memory, because when such memory is freed, its cache is invalidated - as long

* as that memory was never accessed by other cores, except by using this API.

* The first call must be coherent_init(), which initializes the header. If the

* object will be used by multiple cores, next coherent_shared() must be called.

* After that to use that memory, coherent_acquire() must be called, which

* acquires ownership of the object and returns a cached address of the memory.

* After that the user can perform cached access to the memory. To release the

* memory, coherent_release() must be called. The only time when the memory is

* accessed using cache is between those two calls, so only when releasing the

* memory we have to write back and invalidate caches to make sure, that next

* time we acquire this memory, our uncached header access will not be

* overwritten! When memory is not needed any more, typically before freeing the

* memory, coherent_free() should be called.

*

* This structure needs to be embedded at the start of any container to ensure

* container object cache alignment and to minimise non cache access when

* acquiring ownership.

*

* This structure must not be accessed outside of these APIs.

* The shared flag is only set at coherent init and thereafter it's RO.

*/

struct coherent {

union {

struct {

struct k_spinlock lock; /* locking mechanism */

k_spinlock_key_t key; /* lock flags */

};

#ifdef __ZEPHYR__

struct k_mutex mutex;

#endif

};

uint8_t sleep_allowed; /* the object will never be acquired or released

* in atomic context */

uint8_t shared; /* shared on other non coherent cores */

uint16_t core; /* owner core if not shared */

struct list_item list; /* coherent list iteration */

} __coherent;

#if CONFIG_INCOHERENT

# ifdef __ZEPHYR__

BUILD_ASSERT(sizeof(struct coherent) <= DCACHE_LINE_SIZE, "DCACHE_LINE_SIZE too small");

# else

STATIC_ASSERT(sizeof(struct coherent) <= DCACHE_LINE_SIZE, DCACHE_LINE_SIZE_too_small);

# endif

#endif

/* debug address aliases */

#ifdef COHERENT_CHECK_ALIAS

#define ADDR_IS_INCOHERENT(_c) assert(!is_uncached(_c))

#define ADDR_IS_COHERENT(_c) assert(is_uncached(_c))

#else

#define ADDR_IS_INCOHERENT(_c)

#define ADDR_IS_COHERENT(_c)

#endif

/* debug sharing amongst cores */

#ifdef COHERENT_CHECK_NONSHARED_CORES

#define CORE_CHECK_STRUCT_FIELD uint32_t __core; bool __is_shared

#define CORE_CHECK_STRUCT_INIT(_c, is_shared) { (_c)->__core = cpu_get_id(); \

(_c)->__is_shared = is_shared; }

#define CORE_CHECK_STRUCT(_c) { assert(!!(_c)->__is_shared == !!is_uncached(_c)); \

assert(cpu_get_id() == (_c)->__core || (_c)->__is_shared); }

#define CHECK_COHERENT_CORE(_c) assert((_c)->core == cpu_get_id())

#else

#define CORE_CHECK_STRUCT_FIELD

#define CORE_CHECK_STRUCT_INIT(_c, is_shared)

#define CORE_CHECK_STRUCT(_c)

#define CHECK_COHERENT_CORE(_c)

#endif

#ifdef __ZEPHYR__

#define CHECK_ISR() __ASSERT(!k_is_in_isr(), "Attempt to sleep in ISR!")

#define CHECK_SLEEP(_c) __ASSERT((_c)->sleep_allowed, \

"This context hasn't been initialized for sleeping!")

#define CHECK_ATOMIC(_c) __ASSERT(!(_c)->sleep_allowed, \

"This context has been initialized for sleeping!")

#else

#define CHECK_ISR() assert(!k_is_in_isr())

#define CHECK_SLEEP(_c) assert((_c)->sleep_allowed)

#define CHECK_ATOMIC(_c) assert(!(_c)->sleep_allowed)

#endif

#if CONFIG_INCOHERENT

/* When coherent_acquire() is called, we are sure not to have cache for this memory */

__must_check static inline struct coherent __sparse_cache *coherent_acquire(struct coherent *c,

const size_t size)

{

struct coherent __sparse_cache *cc = uncache_to_cache(c);

/* assert if someone passes a cache/local address in here. */

ADDR_IS_COHERENT(c);

CHECK_ATOMIC(c);

/* access the shared coherent object */

if (c->shared) {

CHECK_COHERENT_CORE(c);

c->key = k_spin_lock(&c->lock);

/*

* FIXME: This is wrong. dcache_invalidate_region() only makes

* sense if we assume, that dirty cache lines might exist for

* this object. But in that case those lines could be written

* back here thus overwriting either user data, or the coherent

* header or both. When coherent_acquire() is called it must be

* guaranteed that the object isn't in cache. Before it is

* acquired no cached access to it is allowed. This has to be

* fixed here and on multiple further occasions below.

*/

/* invalidate local copy */

dcache_invalidate_region(cc, size);

}

/* client can now use cached object safely */

return cc;

}

static inline void coherent_release(struct coherent __sparse_cache *c,

const size_t size)

{

/* assert if someone passes a coherent address in here. */

ADDR_IS_INCOHERENT(c);

CHECK_ATOMIC(c);

/* access the local copy of object */

if (c->shared) {

struct coherent *uc = cache_to_uncache(c);

CHECK_COHERENT_CORE(c);

/* wtb and inv local data to coherent object */

dcache_writeback_invalidate_region(c, size);

/* unlock on uncache alias */

k_spin_unlock(&uc->lock, uc->key);

}

}

static inline void *__coherent_init(size_t offset, const size_t size)

{

/*

* Allocate an object with an uncached alias but align size on a cache-

* line boundary to avoid sharing a cache line with the adjacent

* allocation

*/

void *object = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,

ALIGN_UP(size, PLATFORM_DCACHE_ALIGN));

struct coherent *c;

if (!object)

return NULL;

c = (struct coherent *)((uint8_t *)object + offset);

/* TODO static assert if we are not cache aligned */

k_spinlock_init(&c->lock);

c->sleep_allowed = false;

c->shared = false;

c->core = cpu_get_id();

list_init(&c->list);

/* inv local data to coherent object */

dcache_invalidate_region(uncache_to_cache(object), size);

return object;

}

#define coherent_init(type, member) __coherent_init(offsetof(type, member), \

sizeof(type))

/* set the object to shared mode with coherency managed by SW */

static inline void __coherent_shared(struct coherent *c, const size_t size)

{

/* assert if someone passes a cache/local address in here. */

ADDR_IS_COHERENT(c);

CHECK_ATOMIC(c);

c->key = k_spin_lock(&c->lock);

c->shared = true;

dcache_invalidate_region(uncache_to_cache(c), size);

k_spin_unlock(&c->lock, c->key);

}

#define coherent_shared(object, member) __coherent_shared(&(object)->member, \

sizeof(*object))

#ifdef __ZEPHYR__

__must_check static inline struct coherent __sparse_cache *coherent_acquire_thread(

struct coherent *c, const size_t size)

{

struct coherent __sparse_cache *cc = uncache_to_cache(c);

/* assert if someone passes a cache/local address in here. */

ADDR_IS_COHERENT(c);

CHECK_SLEEP(c);

CHECK_ISR();

/* access the shared coherent object */

if (c->shared) {

CHECK_COHERENT_CORE(c);

k_mutex_lock(&c->mutex, K_FOREVER);

/* invalidate local copy */

dcache_invalidate_region(cc, size);

}

/* client can now use cached object safely */

return cc;

}

static inline void coherent_release_thread(struct coherent __sparse_cache *c,

const size_t size)

{

/* assert if someone passes a coherent address in here. */

ADDR_IS_INCOHERENT(c);

CHECK_SLEEP(c);

CHECK_ISR();

/* access the local copy of object */

if (c->shared) {

struct coherent *uc = cache_to_uncache(c);

CHECK_COHERENT_CORE(c);

/* wtb and inv local data to coherent object */

dcache_writeback_invalidate_region(c, size);

/* unlock on uncache alias */

k_mutex_unlock(&uc->mutex);

}

}

static inline void *__coherent_init_thread(size_t offset, const size_t size)

{

/* As above - prevent cache line sharing */

void *object = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,

ALIGN_UP(size, PLATFORM_DCACHE_ALIGN));

struct coherent *c;

if (!object)

return NULL;

c = (struct coherent *)((uint8_t *)object + offset);

/* TODO static assert if we are not cache aligned */

k_mutex_init(&c->mutex);

c->sleep_allowed = true;

c->shared = false;

c->core = cpu_get_id();

list_init(&c->list);

/* inv local data to coherent object */

dcache_invalidate_region(uncache_to_cache(object), size);

return object;

}

#define coherent_init_thread(type, member) __coherent_init_thread(offsetof(type, member), \

sizeof(type))

static inline void __coherent_shared_thread(struct coherent *c, const size_t size)

{

/* assert if someone passes a cache/local address in here. */

ADDR_IS_COHERENT(c);

CHECK_SLEEP(c);

CHECK_ISR();

k_mutex_lock(&c->mutex, K_FOREVER);

c->shared = true;

dcache_invalidate_region(uncache_to_cache(c), size);

k_mutex_unlock(&c->mutex);

}

#define coherent_shared_thread(object, member) __coherent_shared_thread(&(object)->member, \

sizeof(*object))

#endif /* __ZEPHYR__ */

#define coherent_free(object, member) \

do { \

/* assert if someone passes a cache address in here. */ \

ADDR_IS_COHERENT(object); \

/* wtb and inv local data to coherent object */ \

dcache_writeback_invalidate_region(uncache_to_cache(object), \

sizeof(*object)); \

rfree(object); \

} while (0)

#else /* CONFIG_INCOHERENT */

/*

* Coherent devices only require locking to manage shared access.

*/

__must_check static inline struct coherent __sparse_cache *coherent_acquire(struct coherent *c,

const size_t size)

{

if (c->shared)

c->key = k_spin_lock(&c->lock);

return (__sparse_force struct coherent __sparse_cache *)c;

}

static inline void coherent_release(struct coherent __sparse_cache *c,

const size_t size)

{

if (c->shared)

k_spin_unlock(&c->lock, c->key);

}

static inline void *__coherent_init(size_t offset, const size_t size)

{

/* As in CONFIG_INCOHERENT case - prevent cache line sharing */

void *object = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,

ALIGN_UP(size, PLATFORM_DCACHE_ALIGN));

struct coherent *c;

if (!object)

return NULL;

c = (struct coherent *)((uint8_t *)object + offset);

/* TODO static assert if we are not cache aligned */

k_spinlock_init(&c->lock);

c->shared = 0;

c->core = cpu_get_id();

list_init(&c->list);

return object;

}

#define coherent_init(type, member) __coherent_init(offsetof(type, member), \

sizeof(type))

#define coherent_free(object, member) rfree(object)

static inline void __coherent_shared(struct coherent *c, const size_t size)

{

c->key = k_spin_lock(&c->lock);

c->shared = true;

k_spin_unlock(&c->lock, c->key);

}

#define coherent_shared(object, member) __coherent_shared(&(object)->member, \

sizeof(*object))

#ifdef __ZEPHYR__

__must_check static inline struct coherent __sparse_cache *coherent_acquire_thread(

struct coherent *c, const size_t size)

{

if (c->shared)

k_mutex_lock(&c->mutex, K_FOREVER);

return (__sparse_force struct coherent __sparse_cache *)c;

}

static inline void coherent_release_thread(struct coherent __sparse_cache *c,

const size_t size)

{

if (c->shared)

k_mutex_unlock(&c->mutex);

}

static inline void *__coherent_init_thread(size_t offset, const size_t size)

{

/* As above - prevent cache line sharing */

void *object = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,

ALIGN_UP(size, PLATFORM_DCACHE_ALIGN));

struct coherent *c;

if (!object)

return NULL;

c = (struct coherent *)((uint8_t *)object + offset);

/* TODO static assert if we are not cache aligned */

k_mutex_init(&c->mutex);

c->shared = 0;

c->core = cpu_get_id();

list_init(&c->list);

return object;

}

#define coherent_init_thread(type, member) __coherent_init_thread(offsetof(type, member), \

sizeof(type))

static inline void __coherent_shared_thread(struct coherent *c, const size_t size)

{

k_mutex_lock(&c->mutex, K_FOREVER);

c->shared = true;

k_mutex_unlock(&c->mutex);

}

#define coherent_shared_thread(object, member) __coherent_shared_thread(&(object)->member, \

sizeof(*object))

#endif /* __ZEPHYR__ */

#endif /* CONFIG_INCOHERENT */

#ifndef __ZEPHYR__

#define coherent_acquire_thread coherent_acquire

#define coherent_release_thread coherent_release

#define coherent_init_thread coherent_init

#define coherent_shared_thread coherent_shared

#endif

#define coherent_free_thread coherent_free

#define is_coherent_shared(object, member) ((object)->member.shared)

#endif