Preserve GCE bulk creation progress on timeout

CHANGES.rst

@@ -1,5 +1,6 @@
 Changelog
-=========
+==  [Ishu Patel - @ishu7w]
+
 
 Changes in Apache Libcloud 4.0.0
 --------------------------------
@@ -26,6 +27,12 @@ Common
 Compute
 ~~~~~~~
 
+- [GCE] Preserve completed results and pending node names on the exception
+  raised when bulk node creation times out, allowing callers to recover or
+  clean up resources without losing track of them.
+  (GITHUB-1589)
+  [Ishu Patel - @ishu7w]
+
 - [EC2] Allow Elastic IP association with a specific network interface, so
   instances with multiple interfaces can select the required target. Existing
   calls continue to associate by instance ID.

libcloud/compute/drivers/gce.py

@@ -649,6 +649,27 @@ def __repr__(self):
         return '<GCEFailedNode name="{}" error_code="{}">'.format(self.name, self.code)
 
 
+class GCEMultipleNodesTimeoutError(LibcloudError):
+    """
+    Node creation timed out with some results still pending.
+
+    ``nodes`` contains the known results, including :class:`GCEFailedNode`
+    objects when errors were ignored. ``pending_node_names`` identifies nodes
+    whose final state is unknown. These include nodes whose creation has not
+    started and nodes with operations still in progress, which may finish
+    creating after the timeout.
+    No resources are automatically deleted.
+    """
+
+    def __init__(self, timeout, nodes, pending_node_names, driver):
+        self.nodes = nodes
+        self.pending_node_names = pending_node_names
+        super().__init__(
+            "Timeout (%s sec) while waiting for multiple instances" % timeout,
+            driver=driver,
+        )
+
+
 class GCEHealthCheck(UuidMixin):
     """A GCE Http Health Check class."""
 
@@ -5680,7 +5701,11 @@ def ex_create_multiple_nodes(
         :type     ex_service_accounts: ``list``
 
         :keyword  timeout: The number of seconds to wait for all nodes to be
-                           created before timing out.
+                           created before timing out. On timeout,
+                           :class:`GCEMultipleNodesTimeoutError` exposes known
+                           results in ``nodes`` and unresolved names in
+                           ``pending_node_names``. Resources are not deleted;
+                           pending operations may still complete.
         :type     timeout: ``int``
 
         :keyword  description: The description of the node (instance).
@@ -5817,7 +5842,14 @@ def ex_create_multiple_nodes(
 
         while not complete:
             if time.time() - start_time >= timeout:
-                raise Exception("Timeout (%s sec) while waiting for multiple " "instances")
+                raise GCEMultipleNodesTimeoutError(
+                    timeout=timeout,
+                    nodes=[status["node"] for status in status_list if status["node"] is not None],
+                    pending_node_names=[
+                        status["name"] for status in status_list if status["node"] is None
+                    ],
+                    driver=self,
+                )
             complete = True
             time.sleep(poll_interval)
 

libcloud/test/compute/test_gce.py

@@ -46,12 +46,14 @@
     GCEProject,
     GCEFirewall,
     GCENodeImage,
+    GCEFailedNode,
     GCENodeDriver,
     GCESubnetwork,
     GCEHealthCheck,
     GCEBackendService,
     GCEForwardingRule,
     GCETargetHttpProxy,
+    GCEMultipleNodesTimeoutError,
     timestamp_to_datetime,
 )
 from libcloud.test.common.test_google import GoogleTestCase, GoogleAuthMockHttp
@@ -1648,6 +1650,47 @@ def test_ex_create_multiple_nodes_image_family(self):
             ex_image_family="coreos-stable",
         )
 
+    def test_ex_create_multiple_nodes_timeout_preserves_progress(self):
+        image = self.driver.ex_get_image("debian-7")
+        size = self.driver.ex_get_size("n1-standard-1")
+        created = Node("node-id", "lcnode-000", NodeState.RUNNING, [], [], self.driver)
+        failed = GCEFailedNode("lcnode-001", "quota exceeded", "QUOTA_EXCEEDED")
+
+        def create_node(status, node_attrs):
+            if status["name"] == created.name:
+                status["node"] = created
+            elif status["name"] == failed.name:
+                status["node"] = failed
+            else:
+                status["node_response"] = {"selfLink": "pending-operation"}
+
+        with (
+            mock.patch.object(self.driver, "_multi_create_node", side_effect=create_node),
+            mock.patch("libcloud.compute.drivers.gce.time") as clock,
+            mock.patch.object(self.driver, "destroy_node") as destroy,
+        ):
+            clock.time.side_effect = [0, 0, 5]
+            with self.assertRaises(GCEMultipleNodesTimeoutError) as raised:
+                self.driver.ex_create_multiple_nodes(
+                    "lcnode", size, image, 3, timeout=5, poll_interval=0
+                )
+        error = raised.exception
+        self.assertEqual(error.nodes, [created, failed])
+        self.assertEqual(error.pending_node_names, ["lcnode-002"])
+        self.assertIn("5 sec", str(error))
+        self.assertIs(error.driver, self.driver)
+        destroy.assert_not_called()
+
+    def test_ex_create_multiple_nodes_timeout_before_start(self):
+        image = self.driver.ex_get_image("debian-7")
+        size = self.driver.ex_get_size("n1-standard-1")
+        with mock.patch.object(self.driver, "_multi_create_node") as create:
+            with self.assertRaises(GCEMultipleNodesTimeoutError) as raised:
+                self.driver.ex_create_multiple_nodes("lcnode", size, image, 2, timeout=0)
+        self.assertEqual(raised.exception.nodes, [])
+        self.assertEqual(raised.exception.pending_node_names, ["lcnode-000", "lcnode-001"])
+        create.assert_not_called()
+
     def test_ex_create_targethttpproxy(self):
         proxy_name = "web-proxy"
         urlmap_name = "web-map"