diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index dc28e43c7461..2c285a6e5586 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -43,6 +43,20 @@ struct ll_schedule_domain_ops { void (*handler)(void *arg), void *arg); int (*domain_unregister)(struct ll_schedule_domain *domain, struct task *task, uint32_t num_tasks); +#if CONFIG_SOF_USERSPACE_LL + /* + * Initialize the scheduling thread and perform all privileged setup + * (thread creation, timer init, access grants). Called once from + * kernel context before any user-space domain_register() calls. + */ + int (*domain_thread_init)(struct ll_schedule_domain *domain, + struct task *task); + /* Free resources acquired by domain_thread_init(). Called from + * kernel context when the scheduling context is being torn down. + */ + void (*domain_thread_free)(struct ll_schedule_domain *domain, + uint32_t num_tasks); +#endif void (*domain_enable)(struct ll_schedule_domain *domain, int core); void (*domain_disable)(struct ll_schedule_domain *domain, int core); #if CONFIG_CROSS_CORE_STREAM @@ -177,6 +191,31 @@ static inline void domain_task_cancel(struct ll_schedule_domain *domain, domain->ops->domain_task_cancel(domain, task); } +#if CONFIG_SOF_USERSPACE_LL +/* + * Initialize the scheduling thread and do all privileged setup. + * Must be called from kernel context before user-space tasks register. + */ +static inline int domain_thread_init(struct ll_schedule_domain *domain, + struct task *task) +{ + assert(domain->ops->domain_thread_init); + + return domain->ops->domain_thread_init(domain, task); +} + +/* + * Free resources acquired by domain_thread_init(). + * Must be called from kernel context. + */ +static inline void domain_thread_free(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + if (domain->ops->domain_thread_free) + domain->ops->domain_thread_free(domain, num_tasks); +} +#endif + static inline int domain_register(struct ll_schedule_domain *domain, struct task *task, void (*handler)(void *arg), void *arg) diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index bbdcbbecf3b4..9418064beb02 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -158,6 +158,16 @@ struct scheduler_ops { * This operation is optional. */ int (*scheduler_restore)(void *data); + + /** + * Initializes context + * @param data Private data of selected scheduler. + * @param task task that needs to be scheduled + * @return thread that will be used to run the scheduled task + * + * This operation is optional. + */ + struct k_thread *(*scheduler_init_context)(void *data, struct task *task); }; /** \brief Holds information about scheduler. */ @@ -379,6 +389,24 @@ static inline int schedulers_restore(void) return 0; } +/** See scheduler_ops::scheduler_init_context */ +static inline struct k_thread *scheduler_init_context(struct task *task) +{ + struct schedulers *schedulers = *arch_schedulers_get(); + struct schedule_data *sch; + struct list_item *slist; + + assert(schedulers); + + list_for_item(slist, &schedulers->list) { + sch = container_of(slist, struct schedule_data, list); + if (task->type == sch->type && sch->ops->scheduler_init_context) + return sch->ops->scheduler_init_context(sch->data, task); + } + + return NULL; +} + /** * Initializes scheduling task. * @param task Task to be initialized. diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index ebf5aea8d4d2..daace8bac708 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -126,7 +126,8 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) } #endif - dt->handler(dt->arg); + if (dt->handler) + dt->handler(dt->arg); #ifdef CONFIG_SCHEDULE_LL_STATS_LOG cycles1 = k_cycle_get_32(); @@ -289,63 +290,65 @@ static int zephyr_domain_unregister(struct ll_schedule_domain *domain, #else /* CONFIG_SOF_USERSPACE_LL */ -/* User-space implementation for register/unregister */ - -static int zephyr_domain_register_user(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) +/* + * Privileged thread initialization for userspace LL scheduling. + * Creates the scheduling thread, sets up timer, grants access to kernel + * objects. Must be called from kernel context before any user-space + * domain_register() calls. + */ +static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, + struct task *task) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + struct zephyr_domain_thread *dt; char thread_name[] = "ll_thread0"; k_tid_t thread; + int core = task->core; - tr_dbg(&ll_tr, "entry"); + tr_dbg(&ll_tr, "thread_init entry"); - /* domain work only needs registered once on each core */ - if (dt->handler) - return 0; + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; - __ASSERT_NO_MSG(task->core == core); + dt = zephyr_domain->domain_thread + core; - dt->handler = handler; - dt->arg = arg; + /* thread only needs to be created once per core */ + if (dt->ll_thread) + return 0; + + dt->handler = NULL; + dt->arg = NULL; /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ k_sem_init(dt->sem, 0, 10); thread_name[sizeof(thread_name) - 2] = '0' + core; + /* Allocate thread structure dynamically */ + dt->ll_thread = k_object_alloc(K_OBJ_THREAD); if (!dt->ll_thread) { - /* Allocate thread structure dynamically */ - dt->ll_thread = k_object_alloc(K_OBJ_THREAD); - if (!dt->ll_thread) { - tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); - dt->handler = NULL; - dt->arg = NULL; - return -ENOMEM; - } + tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); + return -ENOMEM; + } - thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, - INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, - K_USER, K_FOREVER); + thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, + INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, + K_USER, K_FOREVER); #ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_mask_clear(thread); - k_thread_cpu_mask_enable(thread, core); + k_thread_cpu_mask_clear(thread); + k_thread_cpu_mask_enable(thread, core); #endif - k_thread_name_set(thread, thread_name); + k_thread_name_set(thread, thread_name); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); - k_thread_start(thread); - } + k_thread_start(thread); k_mutex_lock(domain->lock, K_FOREVER); if (!k_timer_user_data_get(zephyr_domain->timer)) { @@ -368,6 +371,43 @@ static int zephyr_domain_register_user(struct ll_schedule_domain *domain, return 0; } +/* + * User-space register: bookkeeping only. The privileged thread setup has + * already been done by domain_thread_init() called from kernel context. + */ +static int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + struct zephyr_domain_thread *dt; + int core; + + tr_dbg(&ll_tr, "register_user entry"); + + if (task->core >= CONFIG_CORE_COUNT) + return -EINVAL; + + core = task->core; + dt = zephyr_domain->domain_thread + core; + + if (!dt->ll_thread) { + tr_err(&ll_tr, "domain_thread_init() not called for core %d", core); + return -EINVAL; + } + + __ASSERT_NO_MSG(!dt->handler || dt->handler == handler); + if (dt->handler) + return 0; + + dt->handler = handler; + dt->arg = arg; + + tr_info(&ll_tr, "task registered on core %d", core); + + return 0; +} + static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, struct task *task, uint32_t num_tasks) { @@ -382,30 +422,58 @@ static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, k_mutex_lock(domain->lock, K_FOREVER); - if (!atomic_read(&domain->total_num_tasks)) { - /* Disable the watchdog */ + zephyr_domain->domain_thread[core].handler = NULL; + + k_mutex_unlock(domain->lock); + + /* + * In this user thread implementation, the timer is left + * running until privileged domain_thread_free() is called + * to clean up resources. + */ + + tr_dbg(&ll_tr, "exit"); + + return 0; +} + +/* + * Free resources acquired by zephyr_domain_thread_init(). + * Stops the timer, aborts the scheduling thread and frees the thread object. + * Must be called from kernel context. + */ +static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); + + /* Still tasks on other cores, only clean up this core's thread */ + k_mutex_lock(domain->lock, K_FOREVER); + + if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { + /* Last task globally: stop the timer and watchdog */ watchdog_disable(core); k_timer_stop(zephyr_domain->timer); k_timer_user_data_set(zephyr_domain->timer, NULL); } - zephyr_domain->domain_thread[core].handler = NULL; + dt->handler = NULL; + dt->arg = NULL; k_mutex_unlock(domain->lock); - tr_info(&ll_tr, "domain->type %d domain->clk %d", - domain->type, domain->clk); - - /* Thread not removed here, only the timer is stopped. - * Thread object cleanup would require k_thread_abort() which cannot - * be safely called from this context. The thread remains allocated - * but dormant until next registration or system shutdown. - */ - - tr_dbg(&ll_tr, "exit"); + if (dt->ll_thread) { + k_thread_abort(dt->ll_thread); + k_object_free(dt->ll_thread); + dt->ll_thread = NULL; + } - return 0; + tr_info(&ll_tr, "thread_free done, core %d", core); } struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) @@ -450,6 +518,8 @@ APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { #ifdef CONFIG_SOF_USERSPACE_LL .domain_register = zephyr_domain_register_user, .domain_unregister = zephyr_domain_unregister_user, + .domain_thread_init = zephyr_domain_thread_init, + .domain_thread_free = zephyr_domain_thread_free, #else .domain_register = zephyr_domain_register, .domain_unregister = zephyr_domain_unregister, diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 575a82d91dda..1c1717d30ac6 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -360,17 +360,7 @@ static int zephyr_ll_task_schedule_common(struct zephyr_ll *sch, struct task *ta ret = domain_register(sch->ll_domain, task, &schedule_ll_callback, sch); if (ret < 0) - tr_err(&ll_tr, "cannot register domain %d", - ret); - -#if CONFIG_SOF_USERSPACE_LL - k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock); - - tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, - zephyr_domain_thread_tid(sch->ll_domain)); -#endif + tr_err(&ll_tr, "cannot register domain %d", ret); return 0; } @@ -509,8 +499,41 @@ static void zephyr_ll_scheduler_free(void *data, uint32_t flags) if (sch->n_tasks) tr_err(&ll_tr, "%u tasks are still active!", sch->n_tasks); + +#if CONFIG_SOF_USERSPACE_LL + domain_thread_free(sch->ll_domain, sch->n_tasks); +#endif } +#if CONFIG_SOF_USERSPACE_LL +struct k_thread *zephyr_ll_init_context(void *data, struct task *task) +{ + struct zephyr_ll *sch = data; + int ret; + + /* + * Use domain_thread_init() for privileged setup (thread creation, + * timer, access grants). domain_register() is now bookkeeping only + * and will be called later from user context when scheduling tasks. + */ + ret = domain_thread_init(sch->ll_domain, task); + if (ret < 0) { + tr_err(&ll_tr, "cannot init_context %d", ret); + return NULL; + } + + assert(!k_is_user_context()); + k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock); + + tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + + return zephyr_domain_thread_tid(sch->ll_domain); +} +#endif + static const struct scheduler_ops zephyr_ll_ops = { .schedule_task = zephyr_ll_task_schedule, .schedule_task_before = zephyr_ll_task_schedule_before, @@ -518,6 +541,9 @@ static const struct scheduler_ops zephyr_ll_ops = { .schedule_task_free = zephyr_ll_task_free, .schedule_task_cancel = zephyr_ll_task_cancel, .scheduler_free = zephyr_ll_scheduler_free, +#if CONFIG_SOF_USERSPACE_LL + .scheduler_init_context = zephyr_ll_init_context, +#endif }; #if CONFIG_SOF_USERSPACE_LL