Add ENABLE_BROKER_STATS to broker.conf

Add instrumentation in the controller to log Passenger memory usage
statistics after every request.

Add ENABLE_BROKER_STATS setting (default false) to
/etc/openshift/broker.conf to enable collecting and logging of these
statistics.

These log entries can be used by the oo-plot-broker-stats command, added in
commit 10d6463, to generate data for
gnuplot.

Author: Ravi Sankar Penta

broker/conf/broker.conf

@@ -241,3 +241,7 @@ MIN_NODE_DISK_BUFFER="5"
 
 # Additional rubygems (space seperated) that will be loaded in the broker's Gemfile.
 # ADDITIONAL_RUBYGEMS=""
+
+# Enables logging of stats like current request id,
+# heap memory, #objects, #symbols for the broker process.
+ENABLE_BROKER_STATS="false"

broker/config/environments/development.rb

@@ -124,6 +124,7 @@
     :auth_user_lookup_only => conf.get_bool("AUTH_USER_LOOKUP_ONLY", false),
     :auth_user_lookup_fail_msg => conf.get("AUTH_USER_LOOKUP_FAIL_MESSAGE", "This cluster is configured for user lookup only. Please contact your system administrator for provisioning your user account."),
     :min_node_disk_buffer => (conf.get("MIN_NODE_DISK_BUFFER", "5")).to_i
+    :broker_stats_enabled => conf.get_bool("ENABLE_BROKER_STATS", "false"),
   }
 
   config.auth = {

broker/config/environments/production.rb

@@ -113,6 +113,7 @@
     :auth_user_lookup_only => conf.get_bool("AUTH_USER_LOOKUP_ONLY", false),
     :auth_user_lookup_fail_msg => conf.get("AUTH_USER_LOOKUP_FAIL_MESSAGE", "This cluster is configured for user lookup only. Please contact your system administrator for provisioning your user account."),
     :min_node_disk_buffer => (conf.get("MIN_NODE_DISK_BUFFER", "5")).to_i
+    :broker_stats_enabled => conf.get_bool("ENABLE_BROKER_STATS", "false"),
   }
 
   config.auth = {

broker/config/environments/test.rb

@@ -122,6 +122,7 @@
     :auth_user_lookup_only => conf.get_bool("AUTH_USER_LOOKUP_ONLY", false),
     :auth_user_lookup_fail_msg => conf.get("AUTH_USER_LOOKUP_FAIL_MESSAGE", "This cluster is configured for user lookup only. Please contact your system administrator for provisioning your user account."),
     :min_node_disk_buffer => (conf.get("MIN_NODE_DISK_BUFFER", "5")).to_i
+    :broker_stats_enabled => conf.get_bool("ENABLE_BROKER_STATS", "false"),
   }
 
   config.auth = {

controller/lib/openshift/controller/api_responses.rb

@@ -47,6 +47,7 @@ def render_error(status, msg, err_code=nil, field=nil, msg_type=nil, messages=ni
             end
             @analytics_tracker.track_event(event_name, @domain, @application, {'request_path' => request.fullpath, 'request_method' => request.method, 'status_code' => status, 'error_code' => err_code, 'error_field' => field})
           end
+          log_broker_stats(request.uuid)
           respond_with reply, :status => reply.status
         end
 
@@ -225,6 +226,7 @@ def render_success(status, type, data, message=nil, result=nil ,extra_messages=n
           else
             log_action(action_log_tag, status, true, message, log_args)
           end
+          log_broker_stats(request.uuid)
           respond_with reply
         end
 
@@ -277,6 +279,17 @@ def extract_node_messages(ex, code=nil, message=nil, field=nil)
             end
             [code, message, messages]
           end
+
+          def log_broker_stats(request_uuid)
+            if Rails.configuration.openshift[:broker_stats_enabled]
+              stats                            = Hash.new
+              stats[:request_id]               = request_uuid
+              stats[:gc_stat]                  = GC::stat
+              stats[:count_objects]            = ObjectSpace.count_objects
+              stats[:count_objects][:T_SYMBOL] = Symbol.all_symbols.size
+              Rails.logger.info("BROKER_STATS => #{stats.to_json}")
+            end
+          end
     end
   end
 end