Execution Hanging Detection

Creating a new system to detect Job hanging and finalize the execution.

Signed-off-by: Rodrigo Nardi <rnardi@netdef.org>

Author: RodrigoMNardi

lib/github/plan_execution/finished.rb

@@ -20,6 +20,7 @@ class Finished
       def initialize(payload)
         @check_suite = CheckSuite.find_by(bamboo_ci_ref: payload['bamboo_ref'])
         @logger = GithubLogger.instance.create('github_plan_execution_finished.log', Logger::INFO)
+        @hanged = payload['hanged'] || false
       end
 
       def finished

lib/github/update_status.rb

@@ -60,6 +60,7 @@ def update_status
       case @status
       when 'in_progress'
         @job.in_progress(@github_check)
+        create_timeout_worker
       when 'success'
         @job.success(@github_check)
         @job.update_execution_time
@@ -81,6 +82,16 @@ def update_status
       [500, 'Internal Server Error']
     end
 
+    def create_timeout_worker
+      Delayed::Job.where('handler LIKE ?', "%TimeoutExecution%args%-%#{@check_suite.id}%")&.delete_all
+
+      logger(Logger::INFO, "CiJobStatus::Update: TimeoutExecution for '#{@check_suite.id}'")
+
+      TimeoutExecution
+        .delay(run_at: 2.hours.from_now.utc, queue: 'timeout_execution')
+        .timeout(@check_suite.id)
+    end
+
     def insert_new_delayed_job
       queue = @job.check_suite.pull_request.github_pr_id % 10
 
@@ -91,7 +102,7 @@ def delete_and_create_delayed_job(queue)
       fetch_delayed_job&.destroy_all
 
       CiJobStatus
-        .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, queue: queue)
+        .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now.utc, queue: queue)
         .update(@job.check_suite.id, @job.id)
     end
 
@@ -117,7 +128,7 @@ def failure
       return failures_stats if @failures.is_a? Array and !@failures.empty?
 
       CiJobFetchTopotestFailures
-        .delay(run_at: 5.minutes.from_now, queue: 'fetch_topotest_failures')
+        .delay(run_at: 5.minutes.from_now.utc, queue: 'fetch_topotest_failures')
         .update(@job.id, 1)
     end
 

lib/github_ci_app.rb

@@ -40,6 +40,7 @@
 
 # Workers
 require_relative '../workers/ci_job_status'
+require_relative '../workers/timeout_execution'
 require_relative '../workers/ci_job_fetch_topotest_failures'
 
 # Slack libs

lib/models/check_suite.rb

@@ -48,4 +48,8 @@ def in_progress?
   def execution_started?
     ci_jobs.where(status: :in_progress).size < 2
   end
+
+  def last_job_updated_at_timer
+    ci_jobs.max_by(&:updated_at)&.updated_at.to_i
+  end
 end

workers/timeout_execution.rb

@@ -0,0 +1,32 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  ci_job_status.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+require_relative '../config/setup'
+
+class TimeoutExecution
+  class << self
+    def timeout(check_suite_id)
+      @logger = GithubLogger.instance.create('timeout_execution_worker.log', Logger::INFO)
+      check_suite = CheckSuite.find(check_suite_id)
+
+      @logger.info("Timeout execution for check_suite_id: #{check_suite_id} -> finished? #{check_suite.finished?}")
+
+      return false if check_suite.finished?
+
+      return false if check_suite.last_job_updated_at_timer > 2.hour.ago.utc
+
+      @logger.info("Calling Github::PlanExecution::Finished.new(#{check_suite.bamboo_ci_ref}).finished")
+
+      Github::PlanExecution::Finished.new({ 'bamboo_ref' => check_suite.bamboo_ci_ref, hanged: true }).finished
+
+      true
+    end
+  end
+end