From b4971267f0d87a9d451f351377f8a7275b2f588c Mon Sep 17 00:00:00 2001
From: DarkIsDude <edouard.comtet@gmail.com>
Date: Wed, 12 Aug 2026 16:31:50 +0200
Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20keep=20serving=20when=20the=20de?=
 =?UTF-8?q?lete=20path=20throws=20synchronously?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A synchronous throw in the data layer while deleting an object escaped
into the promise machinery, where it crossed an async callback that had
already fired. That produced "Callback was already called", raised as an
unhandledRejection, which Node turns into a process exit. One request
therefore killed the pod serving it, and client retries walked the
failure across the deployment until the whole S3 endpoint was down.

Contain it at the delete path: a throw becomes InternalError for that
request and the process keeps serving traffic.

The trigger seen in the field was an unguarded backend-client lookup for
a location removed from the overlay (fixed separately in ARSN-623); this
change stops the next such throw from being fatal.

Issue: CLDSRV-973
---
 lib/services.js | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/lib/services.js b/lib/services.js
index b85fcdaa8d..e1e7ce8181 100644
--- a/lib/services.js
+++ b/lib/services.js
@@ -425,12 +425,25 @@ const services = {
         const objGetInfo = objectMD.location;
         // special case that prevents azure blocks from unecessary deletion
         // will return null if no need
-        return data.protectAzureBlocks(bucketName, objectKey, objGetInfo, log, err => {
-            if (err) {
-                return cb(err);
-            }
-            return deleteMDandData();
-        });
+        // A synchronous throw in the data layer (e.g. no backend client for a location
+        // that was removed from the overlay) must not escape into the promise machinery:
+        // it surfaces there as an unhandledRejection and takes the whole process down.
+        try {
+            return data.protectAzureBlocks(bucketName, objectKey, objGetInfo, log, err => {
+                if (err) {
+                    return cb(err);
+                }
+                return deleteMDandData();
+            });
+        } catch (err) {
+            log.error('deleteObject: error protecting azure blocks', {
+                method: 'services.deleteObject',
+                error: err.message,
+                bucketName,
+                objectKey,
+            });
+            return cb(errors.InternalError);
+        }
     },
 
     /**