|
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | + |
| 3 | +#include <linux/kernel.h> |
| 4 | +#include <linux/uaccess.h> |
| 5 | +#include <linux/sched/deadline.h> |
| 6 | +#include <asm/syscall.h> |
| 7 | +#include <uapi/linux/sched/types.h> |
| 8 | +#include <trace/events/sched.h> |
| 9 | + |
| 10 | +/* |
| 11 | + * Dummy values if not available |
| 12 | + */ |
| 13 | +#ifndef __NR_sched_setscheduler |
| 14 | +#define __NR_sched_setscheduler -__COUNTER__ |
| 15 | +#endif |
| 16 | +#ifndef __NR_sched_setattr |
| 17 | +#define __NR_sched_setattr -__COUNTER__ |
| 18 | +#endif |
| 19 | + |
| 20 | +extern struct rv_monitor rv_deadline; |
| 21 | +/* Initialised when registering the deadline container */ |
| 22 | +extern struct sched_class *rv_ext_sched_class; |
| 23 | + |
| 24 | +/* |
| 25 | + * If both have dummy values, the syscalls are not supported and we don't even |
| 26 | + * need to register the handler. |
| 27 | + */ |
| 28 | +static inline bool should_skip_syscall_handle(void) |
| 29 | +{ |
| 30 | + return __NR_sched_setattr < 0 && __NR_sched_setscheduler < 0; |
| 31 | +} |
| 32 | + |
| 33 | +/* |
| 34 | + * is_supported_type - return true if @type is supported by the deadline monitors |
| 35 | + */ |
| 36 | +static inline bool is_supported_type(u8 type) |
| 37 | +{ |
| 38 | + return type == DL_TASK || type == DL_SERVER_FAIR || type == DL_SERVER_EXT; |
| 39 | +} |
| 40 | + |
| 41 | +/* |
| 42 | + * is_server_type - return true if @type is a supported server |
| 43 | + */ |
| 44 | +static inline bool is_server_type(u8 type) |
| 45 | +{ |
| 46 | + return is_supported_type(type) && type != DL_TASK; |
| 47 | +} |
| 48 | + |
| 49 | +/* |
| 50 | + * Use negative numbers for the server. |
| 51 | + * Currently only one fair server per CPU, may change in the future. |
| 52 | + */ |
| 53 | +#define fair_server_id(cpu) (-cpu) |
| 54 | +#define ext_server_id(cpu) (-cpu - num_possible_cpus()) |
| 55 | +#define NO_SERVER_ID (-2 * num_possible_cpus()) |
| 56 | +/* |
| 57 | + * Get a unique id used for dl entities |
| 58 | + * |
| 59 | + * The cpu is not required for tasks as the pid is used there, if this function |
| 60 | + * is called on a dl_se that for sure corresponds to a task, DL_TASK can be |
| 61 | + * used in place of cpu. |
| 62 | + * We need the cpu for servers as it is provided in the tracepoint and we |
| 63 | + * cannot easily retrieve it from the dl_se (requires the struct rq definition). |
| 64 | + */ |
| 65 | +static inline int get_entity_id(struct sched_dl_entity *dl_se, int cpu, u8 type) |
| 66 | +{ |
| 67 | + if (dl_server(dl_se) && type != DL_TASK) { |
| 68 | + if (type == DL_SERVER_FAIR) |
| 69 | + return fair_server_id(cpu); |
| 70 | + if (type == DL_SERVER_EXT) |
| 71 | + return ext_server_id(cpu); |
| 72 | + return NO_SERVER_ID; |
| 73 | + } |
| 74 | + return dl_task_of(dl_se)->pid; |
| 75 | +} |
| 76 | + |
| 77 | +static inline bool task_is_scx_enabled(struct task_struct *tsk) |
| 78 | +{ |
| 79 | + return IS_ENABLED(CONFIG_SCHED_CLASS_EXT) && |
| 80 | + tsk->sched_class == rv_ext_sched_class; |
| 81 | +} |
| 82 | + |
| 83 | +/* Expand id and target as arguments for da functions */ |
| 84 | +#define EXPAND_ID(dl_se, cpu, type) get_entity_id(dl_se, cpu, type), dl_se |
| 85 | +#define EXPAND_ID_TASK(tsk) get_entity_id(&tsk->dl, task_cpu(tsk), DL_TASK), &tsk->dl |
| 86 | + |
| 87 | +static inline u8 get_server_type(struct task_struct *tsk) |
| 88 | +{ |
| 89 | + if (tsk->policy == SCHED_NORMAL || tsk->policy == SCHED_EXT || |
| 90 | + tsk->policy == SCHED_BATCH || tsk->policy == SCHED_IDLE) |
| 91 | + return task_is_scx_enabled(tsk) ? DL_SERVER_EXT : DL_SERVER_FAIR; |
| 92 | + return DL_OTHER; |
| 93 | +} |
| 94 | + |
| 95 | +static inline int extract_params(struct pt_regs *regs, long id, pid_t *pid_out) |
| 96 | +{ |
| 97 | + size_t size = offsetofend(struct sched_attr, sched_flags); |
| 98 | + struct sched_attr __user *uattr, attr; |
| 99 | + int new_policy = -1, ret; |
| 100 | + unsigned long args[6]; |
| 101 | + |
| 102 | + switch (id) { |
| 103 | + case __NR_sched_setscheduler: |
| 104 | + syscall_get_arguments(current, regs, args); |
| 105 | + *pid_out = args[0]; |
| 106 | + new_policy = args[1]; |
| 107 | + break; |
| 108 | + case __NR_sched_setattr: |
| 109 | + syscall_get_arguments(current, regs, args); |
| 110 | + *pid_out = args[0]; |
| 111 | + uattr = (struct sched_attr __user *)args[1]; |
| 112 | + /* |
| 113 | + * Just copy up to sched_flags, we are not interested after that |
| 114 | + */ |
| 115 | + ret = copy_struct_from_user(&attr, size, uattr, size); |
| 116 | + if (ret) |
| 117 | + return ret; |
| 118 | + if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY) |
| 119 | + return -EINVAL; |
| 120 | + new_policy = attr.sched_policy; |
| 121 | + break; |
| 122 | + default: |
| 123 | + return -EINVAL; |
| 124 | + } |
| 125 | + |
| 126 | + return new_policy & ~SCHED_RESET_ON_FORK; |
| 127 | +} |
| 128 | + |
| 129 | +/* Helper functions requiring DA/HA utilities */ |
| 130 | +#ifdef RV_MON_TYPE |
| 131 | + |
| 132 | +/* |
| 133 | + * get_fair_server - get the fair server associated to a task |
| 134 | + * |
| 135 | + * If the task is a boosted task, the server is available in the task_struct, |
| 136 | + * otherwise grab the dl entity saved for the CPU where the task is enqueued. |
| 137 | + * This function assumes the task is enqueued somewhere. |
| 138 | + */ |
| 139 | +static inline struct sched_dl_entity *get_server(struct task_struct *tsk, u8 type) |
| 140 | +{ |
| 141 | + if (tsk->dl_server && get_server_type(tsk) == type) |
| 142 | + return tsk->dl_server; |
| 143 | + if (type == DL_SERVER_FAIR) |
| 144 | + return da_get_target_by_id(fair_server_id(task_cpu(tsk))); |
| 145 | + if (type == DL_SERVER_EXT) |
| 146 | + return da_get_target_by_id(ext_server_id(task_cpu(tsk))); |
| 147 | + return NULL; |
| 148 | +} |
| 149 | + |
| 150 | +/* |
| 151 | + * Initialise monitors for all tasks and pre-allocate the storage for servers. |
| 152 | + * This is necessary since we don't have access to the servers here and |
| 153 | + * allocation can cause deadlocks from their tracepoints. We can only fill |
| 154 | + * pre-initialised storage from there. |
| 155 | + */ |
| 156 | +static inline int init_storage(bool skip_tasks) |
| 157 | +{ |
| 158 | + struct task_struct *g, *p; |
| 159 | + int cpu; |
| 160 | + |
| 161 | + for_each_possible_cpu(cpu) { |
| 162 | + if (!da_create_empty_storage(fair_server_id(cpu))) |
| 163 | + goto fail; |
| 164 | + if (IS_ENABLED(CONFIG_SCHED_CLASS_EXT) && |
| 165 | + !da_create_empty_storage(ext_server_id(cpu))) |
| 166 | + goto fail; |
| 167 | + } |
| 168 | + |
| 169 | + if (skip_tasks) |
| 170 | + return 0; |
| 171 | + |
| 172 | + read_lock(&tasklist_lock); |
| 173 | + for_each_process_thread(g, p) { |
| 174 | + if (p->policy == SCHED_DEADLINE) { |
| 175 | + if (!da_create_storage(EXPAND_ID_TASK(p), NULL)) { |
| 176 | + read_unlock(&tasklist_lock); |
| 177 | + goto fail; |
| 178 | + } |
| 179 | + } |
| 180 | + } |
| 181 | + read_unlock(&tasklist_lock); |
| 182 | + return 0; |
| 183 | + |
| 184 | +fail: |
| 185 | + da_monitor_destroy(); |
| 186 | + return -ENOMEM; |
| 187 | +} |
| 188 | + |
| 189 | +static void __maybe_unused handle_newtask(void *data, struct task_struct *task, u64 flags) |
| 190 | +{ |
| 191 | + /* Might be superfluous as tasks are not started with this policy.. */ |
| 192 | + if (task->policy == SCHED_DEADLINE) |
| 193 | + da_create_storage(EXPAND_ID_TASK(task), NULL); |
| 194 | +} |
| 195 | + |
| 196 | +static void __maybe_unused handle_exit(void *data, struct task_struct *p, bool group_dead) |
| 197 | +{ |
| 198 | + if (p->policy == SCHED_DEADLINE) |
| 199 | + da_destroy_storage(get_entity_id(&p->dl, DL_TASK, DL_TASK)); |
| 200 | +} |
| 201 | + |
| 202 | +#endif |
0 commit comments