Use a kmem_cache for homa_rpc allocation

Author: johnousterhout

homa_impl.h

@@ -138,6 +138,11 @@ struct homa {
 	 */
 	struct homa_socktab *socktab;
 
+	/**
+	 * @rpc_kmem_cache: Used to allocate homa_rpc structs efficiently.
+	 */
+	struct kmem_cache *rpc_kmem_cache;
+
 #ifndef __STRIP__ /* See strip.py */
 	/**
 	 * @page_pool_mutex: Synchronizes access to any/all of the page_pools

homa_rpc.c

@@ -36,7 +36,7 @@ struct homa_rpc *homa_rpc_alloc_client(struct homa_sock *hsk,
 	struct homa_rpc *crpc;
 	int err;
 
-	crpc = kzalloc(sizeof(*crpc), GFP_KERNEL);
+	crpc = kmem_cache_zalloc(hsk->homa->rpc_kmem_cache, GFP_KERNEL);
 	if (unlikely(!crpc)) {
 		hsk->error_msg = "couldn't allocate memory for client RPC";
 		return ERR_PTR(-ENOMEM);
@@ -95,7 +95,7 @@ struct homa_rpc *homa_rpc_alloc_client(struct homa_sock *hsk,
 error:
 	if (crpc->peer)
 		homa_peer_release(crpc->peer);
-	kfree(crpc);
+	kmem_cache_free(hsk->homa->rpc_kmem_cache, crpc);
 	return ERR_PTR(err);
 }
 
@@ -146,7 +146,7 @@ struct homa_rpc *homa_rpc_alloc_server(struct homa_sock *hsk,
 	}
 
 	/* Initialize fields that don't require the socket lock. */
-	srpc = kzalloc(sizeof(*srpc), GFP_ATOMIC);
+	srpc = kmem_cache_zalloc(hsk->homa->rpc_kmem_cache, GFP_ATOMIC);
 	if (!srpc) {
 		err = -ENOMEM;
 		goto error;
@@ -209,9 +209,11 @@ struct homa_rpc *homa_rpc_alloc_server(struct homa_sock *hsk,
 
 error:
 	homa_bucket_unlock(bucket, id);
-	if (srpc && srpc->peer)
-		homa_peer_release(srpc->peer);
-	kfree(srpc);
+	if (srpc) {
+		if (srpc->peer)
+			homa_peer_release(srpc->peer);
+		kmem_cache_free(hsk->homa->rpc_kmem_cache, srpc);
+	}
 	return ERR_PTR(err);
 }
 
@@ -652,7 +654,7 @@ int homa_rpc_reap(struct homa_sock *hsk, bool reap_all)
 #endif /* See strip.py */
 			rpc->state = 0;
 			rpc->magic = 0;
-			kfree(rpc);
+			kmem_cache_free(hsk->homa->rpc_kmem_cache, rpc);
 		}
 		homa_sock_wakeup_wmem(hsk);
 		tt_record4("reaped %d skbs, %d rpcs; %d skbs remain for port %d",

homa_utils.c

@@ -65,6 +65,13 @@ int homa_init(struct homa *homa)
 	if (!homa->socktab)
 		return -ENOMEM;
 	homa_socktab_init(homa->socktab);
+	homa->rpc_kmem_cache = kmem_cache_create("homa_rpc",
+						 sizeof(struct homa_rpc),
+						 0, SLAB_HWCACHE_ALIGN, NULL);
+	if (!homa->rpc_kmem_cache) {
+		pr_err("Couldn't initialize rpc_kmem_cache\n");
+		return -ENOMEM;
+	}
 #ifndef __STRIP__ /* See strip.py */
 	err = homa_skb_init(homa);
 	if (err) {
@@ -157,6 +164,10 @@ void homa_destroy(struct homa *homa)
 		homa->peertab = NULL;
 	}
 #ifndef __STRIP__ /* See strip.py */
+	if (homa->rpc_kmem_cache) {
+		kmem_cache_destroy(homa->rpc_kmem_cache);
+		homa->rpc_kmem_cache = NULL;
+	}
 
 	homa_skb_cleanup(homa);
 #endif /* See strip.py */

test/mock.c

@@ -28,6 +28,11 @@ extern void      *malloc(size_t size);
 #endif
 extern void      *memcpy(void *dest, const void *src, size_t n);
 
+struct kmem_cache {
+	unsigned int size;
+	int num;
+};
+
 /* The variables below can be set to non-zero values by unit tests in order
  * to simulate error returns from various functions. If bit 0 is set to 1,
  * the next call to the function will fail; bit 1 corresponds to the next
@@ -980,6 +985,41 @@ void *__kmalloc_noprof(size_t size, gfp_t flags)
 	return mock_kmalloc(size, flags);
 }
 
+void *kmem_cache_alloc_noprof(struct kmem_cache *cachep, gfp_t flags)
+{
+	void *result = mock_kmalloc(cachep->size, flags);
+
+	if (result)
+		cachep->num++;
+	return result;
+}
+
+struct kmem_cache *__kmem_cache_create_args(const char *name,
+					    unsigned int object_size,
+					    struct kmem_cache_args *args,
+					    slab_flags_t flags)
+{
+	struct kmem_cache *cache;
+
+	cache = mock_kmalloc(sizeof(*cache), GFP_KERNEL | __GFP_ZERO);
+	if (cache)
+		cache->size = object_size;
+	return cache;
+}
+
+void kmem_cache_destroy(struct kmem_cache *cache)
+{
+	if (cache->num != 0)
+		FAIL(" kmem_cache destroyed with %d live objects", cache->num);
+	kfree(cache);
+}
+
+void kmem_cache_free(struct kmem_cache *cache, void *objp)
+{
+	cache->num--;
+	kfree(objp);
+}
+
 void kvfree(const void *addr)
 {
 	kfree(addr);

test/unit_homa_utils.c

@@ -106,12 +106,21 @@ TEST_F(homa_utils, homa_init__cant_allocate_port_map)
 	EXPECT_EQ(NULL, homa2.socktab);
 	homa_destroy(&homa2);
 }
+TEST_F(homa_utils, homa_init__cant_create_rpc_kmem_cache)
+{
+	struct homa homa2;
+
+	mock_kmalloc_errors = 0x40;
+	EXPECT_EQ(ENOMEM, -homa_init(&homa2));
+	EXPECT_SUBSTR("Couldn't initialize rpc_kmem_cache", mock_printk_output);
+	homa_destroy(&homa2);
+}
 #ifndef __STRIP__ /* See strip.py */
 TEST_F(homa_utils, homa_init__homa_skb_init_failure)
 {
 	struct homa homa2;
 
-	mock_kmalloc_errors = 0x40;
+	mock_kmalloc_errors = 0x80;
 	EXPECT_EQ(ENOMEM, -homa_init(&homa2));
 	EXPECT_SUBSTR("Couldn't initialize skb management (errno 12)",
 		      mock_printk_output);