smb: smbdirect: prepare use of dedicated workqueues for different steps

This is a preparation in order to have global workqueues in
the smbdirect module instead of having the caller to
provide one.

Cc: Steve French <smfrench@gmail.com>
Cc: Tom Talpey <tom@talpey.com>
Cc: Long Li <longli@microsoft.com>
Cc: Namjae Jeon <linkinjeon@kernel.org>
Cc: linux-cifs@vger.kernel.org
Cc: samba-technical@lists.samba.org
Signed-off-by: Stefan Metzmacher <metze@samba.org>
Acked-by: Namjae Jeon <linkinjeon@kernel.org>
Signed-off-by: Steve French <stfrench@microsoft.com>

Author: metze-samba

fs/smb/common/smbdirect/smbdirect_accept.c

@@ -139,7 +139,7 @@ int smbdirect_accept_connect_request(struct smbdirect_socket *sc,
 	 */
 	INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work);
 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
 			 msecs_to_jiffies(sp->negotiate_timeout_msec));
 
 	return 0;
@@ -272,7 +272,7 @@ static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc
 	if (!sc->first_error) {
 		INIT_WORK(&sc->connect.work, smbdirect_accept_negotiate_recv_work);
 		if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)
-			queue_work(sc->workqueue, &sc->connect.work);
+			queue_work(sc->workqueues.accept, &sc->connect.work);
 	}
 	spin_unlock_irqrestore(&sc->connect.lock, flags);
 
@@ -317,7 +317,7 @@ static void smbdirect_accept_negotiate_recv_work(struct work_struct *work)
 	 * order to trigger our next keepalive message.
 	 */
 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
 			 msecs_to_jiffies(sp->keepalive_interval_msec));
 
 	/*
@@ -751,7 +751,7 @@ static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id,
 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;
 		spin_lock_irqsave(&sc->connect.lock, flags);
 		if (!sc->first_error)
-			queue_work(sc->workqueue, &sc->connect.work);
+			queue_work(sc->workqueues.accept, &sc->connect.work);
 		spin_unlock_irqrestore(&sc->connect.lock, flags);
 
 		/*

fs/smb/common/smbdirect/smbdirect_connect.c

@@ -234,7 +234,7 @@ static int smbdirect_connect_rdma_connect(struct smbdirect_socket *sc)
 	 */
 	INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work);
 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
 			 msecs_to_jiffies(sp->rdma_connect_timeout_msec));
 
 	return 0;
@@ -511,7 +511,7 @@ static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc)
 	 * so that the timer will cause a disconnect.
 	 */
 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
 			 msecs_to_jiffies(sp->negotiate_timeout_msec));
 
 	return 0;
@@ -632,7 +632,7 @@ static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc
 	if (!sc->first_error) {
 		INIT_WORK(&sc->connect.work, smbdirect_connect_negotiate_recv_work);
 		if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)
-			queue_work(sc->workqueue, &sc->connect.work);
+			queue_work(sc->workqueues.connect, &sc->connect.work);
 	}
 	spin_unlock_irqrestore(&sc->connect.lock, flags);
 
@@ -680,7 +680,7 @@ static void smbdirect_connect_negotiate_recv_work(struct work_struct *work)
 	 * order to trigger our next keepalive message.
 	 */
 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
 			 msecs_to_jiffies(sp->keepalive_interval_msec));
 
 	/*

fs/smb/common/smbdirect/smbdirect_connection.c

@@ -614,7 +614,7 @@ void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg)
 	sc->statistics.put_receive_buffer++;
 	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);
 
-	queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
+	queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work);
 }
 
 __SMBDIRECT_PRIVATE__
@@ -822,11 +822,11 @@ void smbdirect_connection_idle_timer_work(struct work_struct *work)
 	 * in order to wait for a response
 	 */
 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
 			 msecs_to_jiffies(sp->keepalive_timeout_msec));
 	smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO,
 		"schedule send of empty idle message\n");
-	queue_work(sc->workqueue, &sc->idle.immediate_work);
+	queue_work(sc->workqueues.immediate, &sc->idle.immediate_work);
 }
 
 __SMBDIRECT_PRIVATE__
@@ -878,7 +878,7 @@ static bool smbdirect_connection_request_keep_alive(struct smbdirect_socket *sc)
 		 * Now use the keepalive timeout (instead of keepalive interval)
 		 * in order to wait for a response
 		 */
-		mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+		mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
 				 msecs_to_jiffies(sp->keepalive_timeout_msec));
 		return true;
 	}
@@ -1167,7 +1167,7 @@ int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc,
 		 * get some new recv credits we can grant to
 		 * the peer.
 		 */
-		queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
+		queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work);
 
 		/*
 		 * wait until either the refill work or the peer
@@ -1568,7 +1568,7 @@ void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc)
 	 * order to trigger our next keepalive message.
 	 */
 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
-	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+	mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work,
 			 msecs_to_jiffies(sp->keepalive_interval_msec));
 
 	ib_dma_sync_single_for_cpu(sc->ib.dev,
@@ -1673,7 +1673,7 @@ void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc)
 	if (flags & SMBDIRECT_FLAG_RESPONSE_REQUESTED) {
 		smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO,
 			"schedule send of immediate response\n");
-		queue_work(sc->workqueue, &sc->idle.immediate_work);
+		queue_work(sc->workqueues.immediate, &sc->idle.immediate_work);
 	}
 
 	/*
@@ -1683,7 +1683,7 @@ void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc)
 	if (data_length) {
 		if (current_recv_credits <= (sc->recv_io.credits.target / 4) ||
 		    sc->recv_io.credits.target > old_recv_credit_target)
-			queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
+			queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work);
 
 		smbdirect_connection_reassembly_append_recv_io(sc, recv_io, data_length);
 		wake_up(&sc->recv_io.reassembly.wait_queue);
@@ -1814,7 +1814,7 @@ static void smbdirect_connection_recv_io_refill_work(struct work_struct *work)
 	if (posted > 0) {
 		smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO,
 			"schedule send of an empty message\n");
-		queue_work(sc->workqueue, &sc->idle.immediate_work);
+		queue_work(sc->workqueues.immediate, &sc->idle.immediate_work);
 	}
 }
 

fs/smb/common/smbdirect/smbdirect_socket.c

@@ -242,7 +242,12 @@ int smbdirect_socket_set_custom_workqueue(struct smbdirect_socket *sc,
 	/*
 	 * Remember the callers workqueue
 	 */
-	sc->workqueue = workqueue;
+	sc->workqueues.accept = workqueue;
+	sc->workqueues.connect = workqueue;
+	sc->workqueues.idle = workqueue;
+	sc->workqueues.refill = workqueue;
+	sc->workqueues.immediate = workqueue;
+	sc->workqueues.cleanup = workqueue;
 
 	return 0;
 }
@@ -419,7 +424,7 @@ void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc,
 	 */
 	smbdirect_socket_wake_up_all(sc);
 
-	queue_work(sc->workqueue, &sc->disconnect_work);
+	queue_work(sc->workqueues.cleanup, &sc->disconnect_work);
 }
 
 static void smbdirect_socket_cleanup_work(struct work_struct *work)

fs/smb/common/smbdirect/smbdirect_socket.h

@@ -109,12 +109,19 @@ struct smbdirect_socket {
 	int first_error;
 
 	/*
-	 * This points to the workqueue to
+	 * This points to the workqueues to
 	 * be used for this socket.
 	 * It can be per socket (on the client)
 	 * or point to a global workqueue (on the server)
 	 */
-	struct workqueue_struct *workqueue;
+	struct {
+		struct workqueue_struct *accept;
+		struct workqueue_struct *connect;
+		struct workqueue_struct *idle;
+		struct workqueue_struct *refill;
+		struct workqueue_struct *immediate;
+		struct workqueue_struct *cleanup;
+	} workqueues;
 
 	struct work_struct disconnect_work;