Revert "Use a kmem_cache for homa_rpc allocation"

Performance measurements show that the kmem_cache actually
hurts performance: homa_rpc allocation time increased from
378 cycles without the kmem_cache to 400 cycles with it.

Author: johnousterhout

homa_impl.h

@@ -138,11 +138,6 @@ struct homa {
 	 */
 	struct homa_socktab *socktab;
 
-	/**
-	 * @rpc_kmem_cache: Used to allocate homa_rpc structs efficiently.
-	 */
-	struct kmem_cache *rpc_kmem_cache;
-
 #ifndef __STRIP__ /* See strip.py */
 	/**
 	 * @page_pool_mutex: Synchronizes access to any/all of the page_pools

homa_rpc.c

@@ -36,7 +36,7 @@ struct homa_rpc *homa_rpc_alloc_client(struct homa_sock *hsk,
 	struct homa_rpc *crpc;
 	int err;
 
-	crpc = kmem_cache_zalloc(hsk->homa->rpc_kmem_cache, GFP_KERNEL);
+	crpc = kzalloc(sizeof(*crpc), GFP_KERNEL);
 	if (unlikely(!crpc)) {
 		hsk->error_msg = "couldn't allocate memory for client RPC";
 		return ERR_PTR(-ENOMEM);
@@ -95,7 +95,7 @@ struct homa_rpc *homa_rpc_alloc_client(struct homa_sock *hsk,
 error:
 	if (crpc->peer)
 		homa_peer_release(crpc->peer);
-	kmem_cache_free(hsk->homa->rpc_kmem_cache, crpc);
+	kfree(crpc);
 	return ERR_PTR(err);
 }
 
@@ -146,7 +146,7 @@ struct homa_rpc *homa_rpc_alloc_server(struct homa_sock *hsk,
 	}
 
 	/* Initialize fields that don't require the socket lock. */
-	srpc = kmem_cache_zalloc(hsk->homa->rpc_kmem_cache, GFP_ATOMIC);
+	srpc = kzalloc(sizeof(*srpc), GFP_ATOMIC);
 	if (!srpc) {
 		err = -ENOMEM;
 		goto error;
@@ -209,11 +209,9 @@ struct homa_rpc *homa_rpc_alloc_server(struct homa_sock *hsk,
 
 error:
 	homa_bucket_unlock(bucket, id);
-	if (srpc) {
-		if (srpc->peer)
-			homa_peer_release(srpc->peer);
-		kmem_cache_free(hsk->homa->rpc_kmem_cache, srpc);
-	}
+	if (srpc && srpc->peer)
+		homa_peer_release(srpc->peer);
+	kfree(srpc);
 	return ERR_PTR(err);
 }
 
@@ -654,7 +652,7 @@ int homa_rpc_reap(struct homa_sock *hsk, bool reap_all)
 #endif /* See strip.py */
 			rpc->state = 0;
 			rpc->magic = 0;
-			kmem_cache_free(hsk->homa->rpc_kmem_cache, rpc);
+			kfree(rpc);
 		}
 		homa_sock_wakeup_wmem(hsk);
 		tt_record4("reaped %d skbs, %d rpcs; %d skbs remain for port %d",

homa_utils.c

@@ -65,13 +65,6 @@ int homa_init(struct homa *homa)
 	if (!homa->socktab)
 		return -ENOMEM;
 	homa_socktab_init(homa->socktab);
-	homa->rpc_kmem_cache = kmem_cache_create("homa_rpc",
-						 sizeof(struct homa_rpc),
-						 0, SLAB_HWCACHE_ALIGN, NULL);
-	if (!homa->rpc_kmem_cache) {
-		pr_err("Couldn't initialize rpc_kmem_cache\n");
-		return -ENOMEM;
-	}
 #ifndef __STRIP__ /* See strip.py */
 	err = homa_skb_init(homa);
 	if (err) {
@@ -164,10 +157,6 @@ void homa_destroy(struct homa *homa)
 		homa->peertab = NULL;
 	}
 #ifndef __STRIP__ /* See strip.py */
-	if (homa->rpc_kmem_cache) {
-		kmem_cache_destroy(homa->rpc_kmem_cache);
-		homa->rpc_kmem_cache = NULL;
-	}
 
 	homa_skb_cleanup(homa);
 #endif /* See strip.py */

perf.txt

@@ -2,7 +2,21 @@ This file contains various notes and lessons learned concerning performance
 of the Homa Linux kernel module.  The notes are in reverse chronological
 order.
 
-68. (January 2025) Performance snapshot with and without pacer, using
+69. (March 2026) Use a kmem_cache for homa_rpcs? Tried this, but took it out
+because it made performance slightly worse. Here are measurements of
+cycles/allocation for a 6-node cp_vs_tcp experiment on xl170s:
+
+kmem_cache   no kmem_cache
+416             395
+432             412
+401             381
+357             335
+405             376
+391             368
+-------------------
+400             378    (avg. of avgs)
+
+68. (January 2026) Performance snapshot with and without pacer, using
 c6620 CloudLab nodes, "-w w4 -b 80 -s 20 -n 6". cp_vs_tcp is used unless
 cp_both is indicated.
 

test/mock.c

@@ -28,11 +28,6 @@ extern void      *malloc(size_t size);
 #endif
 extern void      *memcpy(void *dest, const void *src, size_t n);
 
-struct kmem_cache {
-	unsigned int size;
-	int num;
-};
-
 /* The variables below can be set to non-zero values by unit tests in order
  * to simulate error returns from various functions. If bit 0 is set to 1,
  * the next call to the function will fail; bit 1 corresponds to the next
@@ -985,41 +980,6 @@ void *__kmalloc_noprof(size_t size, gfp_t flags)
 	return mock_kmalloc(size, flags);
 }
 
-void *kmem_cache_alloc_noprof(struct kmem_cache *cachep, gfp_t flags)
-{
-	void *result = mock_kmalloc(cachep->size, flags);
-
-	if (result)
-		cachep->num++;
-	return result;
-}
-
-struct kmem_cache *__kmem_cache_create_args(const char *name,
-					    unsigned int object_size,
-					    struct kmem_cache_args *args,
-					    slab_flags_t flags)
-{
-	struct kmem_cache *cache;
-
-	cache = mock_kmalloc(sizeof(*cache), GFP_KERNEL | __GFP_ZERO);
-	if (cache)
-		cache->size = object_size;
-	return cache;
-}
-
-void kmem_cache_destroy(struct kmem_cache *cache)
-{
-	if (cache->num != 0)
-		FAIL(" kmem_cache destroyed with %d live objects", cache->num);
-	kfree(cache);
-}
-
-void kmem_cache_free(struct kmem_cache *cache, void *objp)
-{
-	cache->num--;
-	kfree(objp);
-}
-
 void kvfree(const void *addr)
 {
 	kfree(addr);

test/unit_homa_utils.c

@@ -106,21 +106,12 @@ TEST_F(homa_utils, homa_init__cant_allocate_port_map)
 	EXPECT_EQ(NULL, homa2.socktab);
 	homa_destroy(&homa2);
 }
-TEST_F(homa_utils, homa_init__cant_create_rpc_kmem_cache)
-{
-	struct homa homa2;
-
-	mock_kmalloc_errors = 0x40;
-	EXPECT_EQ(ENOMEM, -homa_init(&homa2));
-	EXPECT_SUBSTR("Couldn't initialize rpc_kmem_cache", mock_printk_output);
-	homa_destroy(&homa2);
-}
 #ifndef __STRIP__ /* See strip.py */
 TEST_F(homa_utils, homa_init__homa_skb_init_failure)
 {
 	struct homa homa2;
 
-	mock_kmalloc_errors = 0x80;
+	mock_kmalloc_errors = 0x40;
 	EXPECT_EQ(ENOMEM, -homa_init(&homa2));
 	EXPECT_SUBSTR("Couldn't initialize skb management (errno 12)",
 		      mock_printk_output);