Add container net alert (#688)

* remove dock and add prometheus

* remove stage hosts

* fix pd key location

* update alert params

Author: anandkumarpatel

ansible/group_vars/all.yml

@@ -5,7 +5,6 @@ github_protocol: https
 
 # ops
 ops_slack_channel_url: https://hooks.slack.com/services/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
-pager_duty_key: testkey
 environment_root: "../environments/{{ env }}/"
 opts_root: "{{ environment_root }}/k8/{{ name }}"
 secrets_root: "{{ environment_root }}/secrets/"
@@ -41,8 +40,6 @@ dockerfile: basic_node/Dockerfile
 docker_network: 172.17.0.0/16
 base_dockerfile: node_base
 
-docker_config: runnable
-
 # slack rooms to send notifications
 slack_token: T029DEC10/B1RSX8LNS/qLLSYEEqkGddohOdE44eDf3j
 slack_channels: [ '#ops' ]

ansible/group_vars/alpha-dock-init.yml

@@ -2,9 +2,6 @@ name: dock-init
 app_name: "{{ name }}"
 app_repo: git@github.com:CodeNow/{{ name }}.git
 
-# for docker role
-docker_config: dock
-
 # consul values
 consul_values:
   - key: "{{ name }}/version"

ansible/group_vars/alpha-prometheus-alerts.yml

@@ -1,7 +1,7 @@
 name: prometheus-alerts
 
 container_image: prom/alertmanager
-container_tag: v0.5.0
+container_tag: v0.6.2
 hosted_ports: ["{{ prometheus_alert_port }}"]
 
 memory_hard_limit: 10G

ansible/roles/datadog/tasks/main.yml

@@ -49,7 +49,6 @@
 
 - name: install network checks for dock services
   become: true
-  when: docker_config == "docks"
   template:
     src=tcp_check.yaml.j2
     dest=/etc/dd-agent/conf.d/tcp_check.yaml

ansible/roles/docker/tasks/main.yml

@@ -15,27 +15,6 @@
     owner=root
     group=root
 
-- name: create core file dir
-  become: true
-  when: docker_config == "runnable" and core_file_dir != "/var/log"
-  tags: coreDump
-  file:
-    path="{{ core_file_dir }}"
-    state=directory
-    owner=root
-    group=root
-    mode=0755
-
-- name: change core dump path
-  become: true
-  when: docker_config == "runnable"
-  tags: coreDump
-  sysctl:
-    name=kernel.core_pattern
-    reload=true
-    state=present
-    value="{{ core_file_dir }}/core.%h.%e.%t"
-
 - name: install aufs with linux-image-extra-{{ ansible_kernel }}
   become: true
   tags: aufs
@@ -70,7 +49,7 @@
 - name: copy docker config file
   become: true
   template:
-    src={{ docker_config }}
+    src=dock
     dest=/etc/default/docker
   register: copied_config
   tags: genDockerConfig

ansible/roles/prometheus-alerts/templates/prometheus-alerts.yml

@@ -11,15 +11,11 @@ data:
 
       slack_api_url: "{{ ops_slack_channel_url }}"
 
-      pagerduty_url: "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
-
     # The root node of the routing tree.
     route:
       # A default receiver
       receiver: slack
 
-      continue: true
-
       routes:
       - match_re:
           reportTo: .*pagerduty.*

ansible/roles/prometheus/files/alerts.conf

@@ -88,3 +88,15 @@ ALERT DockHighLoad
     summary = "({{ $labels.env }}) Dock is experiencing high load host={{ $labels.hostIp }} labels={{ $labels }}",
     description = "ssh {{ $labels.hostIp }} into dock make sure it is responsive, if it is not, unhealthy. `docks unhealthy -e delta {{ $labels.hostIp }}`"
   }
+
+ALERT ContainerUsingTooMuchNetwork
+  IF container_network_transmit_bytes_total{interface="eth0",container_label_type="user-container"} / 1000000 > 200
+  FOR 30m
+  LABELS {
+    reportTo = "pagerduty"
+  }
+  ANNOTATIONS {
+    summary = "({{ $labels.env }}) container is using to much network id={{ $labels.id }} org={{ $labels.githubOrgId }} org name: {{ $labels.container_label_ownerUsername }} ",
+    description = "ssh {{ $labels.hostIp }} and docker rm and docker kill offending container {{ $labels.id }}"
+  }
+

ansible/roles/prometheus/templates/prometheus.yml

@@ -56,7 +56,7 @@ data:
       - source_labels: [__meta_ec2_tag_env]
         target_label:  env
 
-    # pulls server list from ec2 and drops all servers that are not production gamma or a dock
+    # pulls server list from ec2 and drops all servers that are not in env or a dock
     - job_name:         server_info
       # keys to access this region and port of prom
       ec2_sd_configs: