services/nomad: Use per-machine log shippers

Author: the-maldridge

ansible/ansible.cfg

@@ -2,3 +2,7 @@
 inventory = inventory
 nocows = 1
 retry_files_enabled = false
+
+[ssh_connection]
+pipelining = True
+control_path = /tmp/ansible-ssh-%%h-%%p-%%r

ansible/roles/nomad-client/templates/40-client.hcl

@@ -21,6 +21,11 @@ client {
     path = "/var/log"
     read_only = false
   }
+
+  host_volume "dockersocket" {
+    path = "/run/docker.sock"
+    read_only = true
+  }
 {% for volume in nomad_host_volumes|default([]) %}
 
   host_volume "{{volume.name}}" {
@@ -34,3 +39,9 @@ vault {
   enabled = true
   address = "http://active.vault.service.consul:8200"
 }
+
+plugin "docker" {
+  config {
+    extra_labels = ["*"]
+  }
+}

ansible/roles/vault/tasks/main.yml

@@ -4,14 +4,6 @@
     pkg: vault
     state: present
 
-- name: Enforce ownership on /etc/vault
-  file:
-    path: /etc/vault
-    state: directory
-    owner: _vault
-    group: _vault
-    mode: 0750
-
 - name: Install Storage config
   template:
     src: 25-storage.hcl.j2

services/nomad/apps/debuginfod.nomad

@@ -81,37 +81,5 @@ job "debuginfod" {
         delay = "30s"
       }
     }
-    task "promtail" {
-      driver = "docker"
-
-      config {
-        image = "grafana/promtail:2.1.0"
-        args = ["-config.file=/local/promtail.yml"]
-      }
-
-      template {
-                data = <<EOT
----
-server:
-  disable: true
-clients:
-  - url: http://loki.service.consul:3100/loki/api/v1/push
-positions:
-  filename: /alloc/positions.yaml
-scrape_configs:
-  - job_name: debuginfod
-    static_configs:
-      - targets:
-        - localhost
-        labels:
-          __path__: /alloc/logs/debuginfod*
-          nomad_namespace: "{{ env "NOMAD_NAMESPACE" }}"
-          nomad_job: "debuginfod"
-          nomad_group: "{{ env "NOMAD_GROUP_NAME" }}"
-          nomad_task: "{{ env "NOMAD_TASK_NAME" }}"
-EOT
-        destination = "local/promtail.yml"
-      }
-    }
   }
 }

services/nomad/build/buildsync-aarch64.nomad

@@ -51,37 +51,5 @@ EOF
         destination = "local/known_hosts"
       }
     }
-    task "promtail" {
-      driver = "docker"
-
-      config {
-        image = "grafana/promtail:2.1.0"
-        args = ["-config.file=/local/promtail.yml"]
-      }
-
-      template {
-                data = <<EOT
----
-server:
-  disable: true
-clients:
-  - url: http://loki.service.consul:3100/loki/api/v1/push
-positions:
-  filename: /alloc/positions.yaml
-scrape_configs:
-  - job_name: buildsync-aarch64
-    static_configs:
-      - targets:
-        - localhost
-        labels:
-          __path__: /alloc/logs/rsync*
-          nomad_namespace: "{{ env "NOMAD_NAMESPACE" }}"
-          nomad_job: "buildsync-aarch64"
-          nomad_group: "{{ env "NOMAD_GROUP_NAME" }}"
-          nomad_task: "{{ env "NOMAD_TASK_NAME" }}"
-EOT
-        destination = "local/promtail.yml"
-      }
-    }
   }
 }

services/nomad/build/buildsync-dist.nomad

@@ -69,37 +69,5 @@ EOF
         destination = "local/known_hosts"
       }
     }
-    task "promtail" {
-      driver = "docker"
-
-      config {
-        image = "grafana/promtail:2.1.0"
-        args = ["-config.file=/local/promtail.yml"]
-      }
-
-      template {
-                data = <<EOT
----
-server:
-  disable: true
-clients:
-  - url: http://loki.service.consul:3100/loki/api/v1/push
-positions:
-  filename: /alloc/positions.yaml
-scrape_configs:
-  - job_name: buildsync-dist
-    static_configs:
-      - targets:
-        - localhost
-        labels:
-          __path__: /alloc/logs/rsync*
-          nomad_namespace: "{{ env "NOMAD_NAMESPACE" }}"
-          nomad_job: "buildsync-dist"
-          nomad_group: "{{ env "NOMAD_GROUP_NAME" }}"
-          nomad_task: "{{ env "NOMAD_TASK_NAME" }}"
-EOT
-        destination = "local/promtail.yml"
-      }
-    }
   }
 }

services/nomad/build/buildsync-musl.nomad

@@ -51,37 +51,5 @@ EOF
         destination = "local/known_hosts"
       }
     }
-    task "promtail" {
-      driver = "docker"
-
-      config {
-        image = "grafana/promtail:2.1.0"
-        args = ["-config.file=/local/promtail.yml"]
-      }
-
-      template {
-                data = <<EOT
----
-server:
-  disable: true
-clients:
-  - url: http://loki.service.consul:3100/loki/api/v1/push
-positions:
-  filename: /alloc/positions.yaml
-scrape_configs:
-  - job_name: buildsync-musl
-    static_configs:
-      - targets:
-        - localhost
-        labels:
-          __path__: /alloc/logs/rsync*
-          nomad_namespace: "{{ env "NOMAD_NAMESPACE" }}"
-          nomad_job: "buildsync-musl"
-          nomad_group: "{{ env "NOMAD_GROUP_NAME" }}"
-          nomad_task: "{{ env "NOMAD_TASK_NAME" }}"
-EOT
-        destination = "local/promtail.yml"
-      }
-    }
   }
 }

services/nomad/monitoring/grafana.nomad

@@ -108,6 +108,10 @@ EOT
         image = "ghcr.io/netauth/ldap:v0.2.3"
       }
 
+      resources {
+        memory = 100
+      }
+
       env {
         NETAUTH_LOGLEVEL = "trace"
       }

services/nomad/monitoring/vector.nomad

@@ -0,0 +1,82 @@
+job "vector" {
+  datacenters = [
+    "VOID",
+    "VOID-MX",
+    "VOID-PROXY",
+    "VOID-MIRROR",
+    "VOID-CONTROL",
+  ]
+  namespace   = "monitoring"
+  type        = "system"
+  priority    = 99
+
+  group "vector" {
+    network {
+      mode = "bridge"
+      port "metrics" { to = 8088 }
+    }
+
+    volume "dockersocket" {
+      type      = "host"
+      source    = "dockersocket"
+      read_only = true
+    }
+
+    service {
+      name         = "vector"
+      port         = "8088"
+      address_mode = "alloc"
+    }
+
+    task "vector" {
+      driver = "docker"
+
+      config {
+        image = "timberio/vector:0.13.1-debian"
+        args  = ["-c", "/local/vector.yaml"]
+        ports = ["metrics"]
+      }
+
+      volume_mount {
+        volume      = "dockersocket"
+        destination = "/var/run/docker.sock"
+        read_only   = true
+      }
+
+      template {
+                data = <<EOF
+---
+sources:
+  docker:
+    type: docker_logs
+    exclude_containers:
+      - "vector-"
+      - "loki-"
+sinks:
+  loki:
+    type: loki
+    inputs:
+      - docker
+    endpoint: http://loki.service.consul:3100
+    encoding:
+      codec: text
+    healthcheck:
+      enabled: false
+    out_of_order_action: drop
+    labels:
+      nomad_namespace: "{{ label.com\\.hashicorp\\.nomad\\.namespace }}"
+      nomad_job: "{{ label.com\\.hashicorp\\.nomad\\.job_name }}"
+      nomad_group: "{{ label.com\\.hashicorp\\.nomad\\.task_group_name }}"
+      nomad_task: "{{ label.com\\.hashicorp\\.nomad\\.task_name }}"
+      nomad_node: "{{ label.com\\.hashicorp\\.nomad\\.node_name }}"
+      nomad_alloc: "{{ label.com\\.hashicorp\\.nomad\\.alloc_id }}"
+EOF
+        left_delimiter  = "///1"
+        right_delimiter = "///2"
+
+        destination = "local/vector.yaml"
+      }
+
+    }
+  }
+}