diff --git a/mlx/backend/metal/device.h b/mlx/backend/metal/device.h
index 2d22283351..0ca70b5377 100644
--- a/mlx/backend/metal/device.h
+++ b/mlx/backend/metal/device.h
@@ -62,8 +62,10 @@ class MLX_API CommandEncoder {
 
   template <typename Vec, typename = std::enable_if_t<is_vector_v<Vec>>>
   void set_vector_bytes(const Vec& vec, size_t nelems, int idx) {
-    get_command_encoder()->setBytes(
-        vec.data(), nelems * sizeof(typename Vec::value_type), idx);
+    if (nelems > 0) {
+      get_command_encoder()->setBytes(
+          vec.data(), nelems * sizeof(typename Vec::value_type), idx);
+    }
   }
   template <typename Vec, typename = std::enable_if_t<is_vector_v<Vec>>>
   void set_vector_bytes(const Vec& vec, int idx) {