Skip to content

Commit 7eb7e8a

Browse files
axboeaxboe
committed
Merge branch 'for-7.1/io_uring' into for-next
* for-7.1/io_uring: io_uring: unify getting ctx from passed in file descriptor io_uring/register: don't get a reference to the registered ring fd io_uring/tctx: clean up __io_uring_add_tctx_node() error handling io_uring/tctx: have io_uring_alloc_task_context() return tctx
2 parents 485f07e + c5e9f6a commit 7eb7e8a

9 files changed

Lines changed: 101 additions & 93 deletions

File tree

‎io_uring/bpf-ops.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ static int bpf_io_reg(void *kdata, struct bpf_link *link)
181181
struct file *file;
182182
int ret = -EBUSY;
183183

184-
file = io_uring_register_get_file(ops->ring_fd, false);
184+
file = io_uring_ctx_get_file(ops->ring_fd, false);
185185
if (IS_ERR(file))
186186
return PTR_ERR(file);
187187
ctx = file->private_data;

‎io_uring/io_uring.c‎

Lines changed: 36 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2546,39 +2546,54 @@ static int io_get_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
25462546
#endif
25472547
}
25482548

2549-
SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
2550-
u32, min_complete, u32, flags, const void __user *, argp,
2551-
size_t, argsz)
2549+
/*
2550+
* Given an 'fd' value, return the ctx associated with if. If 'registered' is
2551+
* true, then the registered index is used. Otherwise, the normal fd table.
2552+
* Caller must call fput() on the returned file if it isn't a registered file,
2553+
* unless it's an ERR_PTR.
2554+
*/
2555+
struct file *io_uring_ctx_get_file(unsigned int fd, bool registered)
25522556
{
2553-
struct io_ring_ctx *ctx;
25542557
struct file *file;
2555-
long ret;
2556-
2557-
if (unlikely(flags & ~IORING_ENTER_FLAGS))
2558-
return -EINVAL;
25592558

2560-
/*
2561-
* Ring fd has been registered via IORING_REGISTER_RING_FDS, we
2562-
* need only dereference our task private array to find it.
2563-
*/
2564-
if (flags & IORING_ENTER_REGISTERED_RING) {
2559+
if (registered) {
2560+
/*
2561+
* Ring fd has been registered via IORING_REGISTER_RING_FDS, we
2562+
* need only dereference our task private array to find it.
2563+
*/
25652564
struct io_uring_task *tctx = current->io_uring;
25662565

25672566
if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
2568-
return -EINVAL;
2567+
return ERR_PTR(-EINVAL);
25692568
fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
25702569
file = tctx->registered_rings[fd];
2571-
if (unlikely(!file))
2572-
return -EBADF;
25732570
} else {
25742571
file = fget(fd);
2575-
if (unlikely(!file))
2576-
return -EBADF;
2577-
ret = -EOPNOTSUPP;
2578-
if (unlikely(!io_is_uring_fops(file)))
2579-
goto out;
25802572
}
25812573

2574+
if (unlikely(!file))
2575+
return ERR_PTR(-EBADF);
2576+
if (io_is_uring_fops(file))
2577+
return file;
2578+
fput(file);
2579+
return ERR_PTR(-EOPNOTSUPP);
2580+
}
2581+
2582+
2583+
SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
2584+
u32, min_complete, u32, flags, const void __user *, argp,
2585+
size_t, argsz)
2586+
{
2587+
struct io_ring_ctx *ctx;
2588+
struct file *file;
2589+
long ret;
2590+
2591+
if (unlikely(flags & ~IORING_ENTER_FLAGS))
2592+
return -EINVAL;
2593+
2594+
file = io_uring_ctx_get_file(fd, flags & IORING_ENTER_REGISTERED_RING);
2595+
if (IS_ERR(file))
2596+
return PTR_ERR(file);
25822597
ctx = file->private_data;
25832598
ret = -EBADFD;
25842599
/*

‎io_uring/io_uring.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ void io_req_track_inflight(struct io_kiocb *req);
185185
struct file *io_file_get_normal(struct io_kiocb *req, int fd);
186186
struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
187187
unsigned issue_flags);
188+
struct file *io_uring_ctx_get_file(unsigned int fd, bool registered);
188189

189190
void io_req_task_queue(struct io_kiocb *req);
190191
void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw);

‎io_uring/register.c‎

Lines changed: 3 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -946,40 +946,6 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
946946
return ret;
947947
}
948948

949-
/*
950-
* Given an 'fd' value, return the ctx associated with if. If 'registered' is
951-
* true, then the registered index is used. Otherwise, the normal fd table.
952-
* Caller must call fput() on the returned file, unless it's an ERR_PTR.
953-
*/
954-
struct file *io_uring_register_get_file(unsigned int fd, bool registered)
955-
{
956-
struct file *file;
957-
958-
if (registered) {
959-
/*
960-
* Ring fd has been registered via IORING_REGISTER_RING_FDS, we
961-
* need only dereference our task private array to find it.
962-
*/
963-
struct io_uring_task *tctx = current->io_uring;
964-
965-
if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
966-
return ERR_PTR(-EINVAL);
967-
fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
968-
file = tctx->registered_rings[fd];
969-
if (file)
970-
get_file(file);
971-
} else {
972-
file = fget(fd);
973-
}
974-
975-
if (unlikely(!file))
976-
return ERR_PTR(-EBADF);
977-
if (io_is_uring_fops(file))
978-
return file;
979-
fput(file);
980-
return ERR_PTR(-EOPNOTSUPP);
981-
}
982-
983949
static int io_uring_register_send_msg_ring(void __user *arg, unsigned int nr_args)
984950
{
985951
struct io_uring_sqe sqe;
@@ -1034,7 +1000,7 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
10341000
if (fd == -1)
10351001
return io_uring_register_blind(opcode, arg, nr_args);
10361002

1037-
file = io_uring_register_get_file(fd, use_registered_ring);
1003+
file = io_uring_ctx_get_file(fd, use_registered_ring);
10381004
if (IS_ERR(file))
10391005
return PTR_ERR(file);
10401006
ctx = file->private_data;
@@ -1046,6 +1012,7 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
10461012
ctx->buf_table.nr, ret);
10471013
mutex_unlock(&ctx->uring_lock);
10481014

1049-
fput(file);
1015+
if (!use_registered_ring)
1016+
fput(file);
10501017
return ret;
10511018
}

‎io_uring/register.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,5 @@
44

55
int io_eventfd_unregister(struct io_ring_ctx *ctx);
66
int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id);
7-
struct file *io_uring_register_get_file(unsigned int fd, bool registered);
87

98
#endif

‎io_uring/rsrc.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1326,7 +1326,7 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)
13261326
return -EINVAL;
13271327

13281328
registered_src = (buf.flags & IORING_REGISTER_SRC_REGISTERED) != 0;
1329-
file = io_uring_register_get_file(buf.src_fd, registered_src);
1329+
file = io_uring_ctx_get_file(buf.src_fd, registered_src);
13301330
if (IS_ERR(file))
13311331
return PTR_ERR(file);
13321332

@@ -1348,7 +1348,8 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)
13481348
if (src_ctx != ctx)
13491349
mutex_unlock(&src_ctx->uring_lock);
13501350

1351-
fput(file);
1351+
if (!registered_src)
1352+
fput(file);
13521353
return ret;
13531354
}
13541355

‎io_uring/sqpoll.c‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,7 @@ __cold int io_sq_offload_create(struct io_ring_ctx *ctx,
458458
return -EINVAL;
459459
}
460460
if (ctx->flags & IORING_SETUP_SQPOLL) {
461+
struct io_uring_task *tctx;
461462
struct task_struct *tsk;
462463
struct io_sq_data *sqd;
463464
bool attached;
@@ -524,8 +525,13 @@ __cold int io_sq_offload_create(struct io_ring_ctx *ctx,
524525
rcu_assign_pointer(sqd->thread, tsk);
525526
mutex_unlock(&sqd->lock);
526527

528+
ret = 0;
527529
get_task_struct(tsk);
528-
ret = io_uring_alloc_task_context(tsk, ctx);
530+
tctx = io_uring_alloc_task_context(tsk, ctx);
531+
if (!IS_ERR(tctx))
532+
tsk->io_uring = tctx;
533+
else
534+
ret = PTR_ERR(tctx);
529535
wake_up_new_task(tsk);
530536
if (ret)
531537
goto err;

‎io_uring/tctx.c‎

Lines changed: 48 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -74,60 +74,85 @@ void __io_uring_free(struct task_struct *tsk)
7474
}
7575
}
7676

77-
__cold int io_uring_alloc_task_context(struct task_struct *task,
78-
struct io_ring_ctx *ctx)
77+
__cold struct io_uring_task *io_uring_alloc_task_context(struct task_struct *task,
78+
struct io_ring_ctx *ctx)
7979
{
8080
struct io_uring_task *tctx;
8181
int ret;
8282

8383
tctx = kzalloc_obj(*tctx);
8484
if (unlikely(!tctx))
85-
return -ENOMEM;
85+
return ERR_PTR(-ENOMEM);
8686

8787
ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
8888
if (unlikely(ret)) {
8989
kfree(tctx);
90-
return ret;
90+
return ERR_PTR(ret);
9191
}
9292

9393
tctx->io_wq = io_init_wq_offload(ctx, task);
9494
if (IS_ERR(tctx->io_wq)) {
9595
ret = PTR_ERR(tctx->io_wq);
9696
percpu_counter_destroy(&tctx->inflight);
9797
kfree(tctx);
98-
return ret;
98+
return ERR_PTR(ret);
9999
}
100100

101101
tctx->task = task;
102102
xa_init(&tctx->xa);
103103
init_waitqueue_head(&tctx->wait);
104104
atomic_set(&tctx->in_cancel, 0);
105105
atomic_set(&tctx->inflight_tracked, 0);
106-
task->io_uring = tctx;
107106
init_llist_head(&tctx->task_list);
108107
init_task_work(&tctx->task_work, tctx_task_work);
108+
return tctx;
109+
}
110+
111+
static int io_tctx_install_node(struct io_ring_ctx *ctx,
112+
struct io_uring_task *tctx)
113+
{
114+
struct io_tctx_node *node;
115+
int ret;
116+
117+
if (xa_load(&tctx->xa, (unsigned long)ctx))
118+
return 0;
119+
120+
node = kmalloc_obj(*node);
121+
if (!node)
122+
return -ENOMEM;
123+
node->ctx = ctx;
124+
node->task = current;
125+
126+
ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
127+
node, GFP_KERNEL));
128+
if (ret) {
129+
kfree(node);
130+
return ret;
131+
}
132+
133+
mutex_lock(&ctx->tctx_lock);
134+
list_add(&node->ctx_node, &ctx->tctx_list);
135+
mutex_unlock(&ctx->tctx_lock);
109136
return 0;
110137
}
111138

112139
int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
113140
{
114141
struct io_uring_task *tctx = current->io_uring;
115-
struct io_tctx_node *node;
116142
int ret;
117143

118144
if (unlikely(!tctx)) {
119-
ret = io_uring_alloc_task_context(current, ctx);
120-
if (unlikely(ret))
121-
return ret;
145+
tctx = io_uring_alloc_task_context(current, ctx);
146+
if (IS_ERR(tctx))
147+
return PTR_ERR(tctx);
122148

123-
tctx = current->io_uring;
124149
if (ctx->int_flags & IO_RING_F_IOWQ_LIMITS_SET) {
125150
unsigned int limits[2] = { ctx->iowq_limits[0],
126151
ctx->iowq_limits[1], };
127152

128153
ret = io_wq_max_workers(tctx->io_wq, limits);
129154
if (ret)
130-
return ret;
155+
goto err_free;
131156
}
132157
}
133158

@@ -138,25 +163,19 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
138163
*/
139164
if (tctx->io_wq)
140165
io_wq_set_exit_on_idle(tctx->io_wq, false);
141-
if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
142-
node = kmalloc_obj(*node);
143-
if (!node)
144-
return -ENOMEM;
145-
node->ctx = ctx;
146-
node->task = current;
147-
148-
ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
149-
node, GFP_KERNEL));
150-
if (ret) {
151-
kfree(node);
152-
return ret;
153-
}
154166

155-
mutex_lock(&ctx->tctx_lock);
156-
list_add(&node->ctx_node, &ctx->tctx_list);
157-
mutex_unlock(&ctx->tctx_lock);
167+
ret = io_tctx_install_node(ctx, tctx);
168+
if (!ret) {
169+
current->io_uring = tctx;
170+
return 0;
158171
}
159-
return 0;
172+
if (!current->io_uring) {
173+
err_free:
174+
io_wq_put_and_exit(tctx->io_wq);
175+
percpu_counter_destroy(&tctx->inflight);
176+
kfree(tctx);
177+
}
178+
return ret;
160179
}
161180

162181
int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx)

‎io_uring/tctx.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ struct io_tctx_node {
66
struct io_ring_ctx *ctx;
77
};
88

9-
int io_uring_alloc_task_context(struct task_struct *task,
10-
struct io_ring_ctx *ctx);
9+
struct io_uring_task *io_uring_alloc_task_context(struct task_struct *task,
10+
struct io_ring_ctx *ctx);
1111
void io_uring_del_tctx_node(unsigned long index);
1212
int __io_uring_add_tctx_node(struct io_ring_ctx *ctx);
1313
int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx);

0 commit comments

Comments
 (0)