diff --git a/Vagrantfile b/Vagrantfile index 99eace0e39..736e46bd70 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -14,6 +14,7 @@ Vagrant.configure("2") do |config| config.vm.network "forwarded_port", guest: 3000, host: 3000, host_ip: "0.0.0.0", id: "grafana", auto_correct: true config.vm.network "forwarded_port", guest: 3100, host: 3100, host_ip: "0.0.0.0", id: "loki", auto_correct: true config.vm.network "forwarded_port", guest: 9080, host: 9080, host_ip: "0.0.0.0", id: "promtail", auto_correct: true + config.vm.network "forwarded_port", guest: 9090, host: 9090, host_ip: "0.0.0.0", id: "prometheus", auto_correct: true config.ssh.insert_key = true diff --git a/ansible/roles/monitoring/defaults/main.yml b/ansible/roles/monitoring/defaults/main.yml index 76fb4745d6..2524c6e8fe 100644 --- a/ansible/roles/monitoring/defaults/main.yml +++ b/ansible/roles/monitoring/defaults/main.yml @@ -5,23 +5,31 @@ monitoring_compose_project_name: devops-monitoring monitoring_loki_version: "3.0.0" monitoring_promtail_version: "3.0.0" monitoring_grafana_version: "12.3.1" +monitoring_prometheus_version: "v3.9.0" monitoring_loki_port: 3100 monitoring_promtail_port: 9080 monitoring_grafana_port: 3000 +monitoring_prometheus_port: 9090 monitoring_app_port: 8000 monitoring_app_internal_port: 8000 monitoring_loki_retention_period: "168h" +monitoring_prometheus_retention_days: 15 +monitoring_prometheus_retention_size: "10GB" +monitoring_prometheus_scrape_interval: "15s" + monitoring_grafana_admin_user: admin -monitoring_grafana_admin_password: ChangeMe_Lab07! -monitoring_grafana_datasource_uid: loki -monitoring_grafana_datasource_name: Loki +monitoring_grafana_admin_password: ChangeMe_Lab08! +monitoring_loki_datasource_uid: loki +monitoring_loki_datasource_name: Loki +monitoring_prometheus_datasource_uid: prometheus +monitoring_prometheus_datasource_name: Prometheus monitoring_app_service_name: app-python monitoring_app_container_name: devops-python monitoring_app_label: devops-python -monitoring_app_image: devops-info-service:lab07 +monitoring_app_image: devops-info-service:lab08 monitoring_app_source_dir: "{{ playbook_dir }}/../../app_python" monitoring_app_source_files: - .dockerignore @@ -33,16 +41,36 @@ monitoring_app_env: PORT: "{{ monitoring_app_internal_port | string }}" DEBUG: "false" +monitoring_prometheus_targets: + - job: prometheus + targets: + - localhost:9090 + - job: app + targets: + - "{{ monitoring_app_service_name }}:{{ monitoring_app_internal_port }}" + path: /metrics + - job: loki + targets: + - loki:3100 + path: /metrics + - job: grafana + targets: + - grafana:3000 + path: /metrics + monitoring_resource_limits: loki: limits: { cpus: '1.0', memory: 1G } reservations: { cpus: '0.25', memory: 256M } promtail: - limits: { cpus: '0.75', memory: 512M } + limits: { cpus: '0.5', memory: 512M } reservations: { cpus: '0.10', memory: 128M } grafana: + limits: { cpus: '0.5', memory: 512M } + reservations: { cpus: '0.10', memory: 128M } + prometheus: limits: { cpus: '1.0', memory: 1G } reservations: { cpus: '0.25', memory: 256M } app_python: - limits: { cpus: '0.75', memory: 512M } + limits: { cpus: '0.5', memory: 256M } reservations: { cpus: '0.10', memory: 128M } diff --git a/ansible/roles/monitoring/files/lab08-metrics.json b/ansible/roles/monitoring/files/lab08-metrics.json new file mode 100644 index 0000000000..b05c0fa86a --- /dev/null +++ b/ansible/roles/monitoring/files/lab08-metrics.json @@ -0,0 +1,454 @@ +{ + "id": null, + "uid": "lab08-prometheus-metrics", + "title": "Lab08 - Prometheus Metrics Overview", + "tags": [ + "lab08", + "prometheus", + "metrics", + "observability" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Request Rate by Endpoint", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (endpoint) (rate(http_requests_total{endpoint!=\"/metrics\"}[5m]))", + "legendFormat": "{{endpoint}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Error Rate (5xx)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(http_requests_total{status_code=~\"5..\"}[5m]))", + "legendFormat": "5xx errors/sec", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Request Duration p95 by Endpoint", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, endpoint) (rate(http_request_duration_seconds_bucket{endpoint!=\"/metrics\"}[5m])))", + "legendFormat": "{{endpoint}} p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Active Requests", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(http_requests_in_progress)", + "legendFormat": "in-flight", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "Application Uptime", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max(devops_info_uptime_seconds)", + "legendFormat": "uptime", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + } + }, + { + "id": 6, + "type": "heatmap", + "title": "Request Duration Heatmap", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (le) (rate(http_request_duration_seconds_bucket{endpoint!=\"/metrics\"}[5m]))", + "legendFormat": "{{le}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "calculate": false, + "legend": { + "show": false + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "unit": "s" + } + } + }, + { + "id": 7, + "type": "piechart", + "title": "Status Code Distribution", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (status_code) (rate(http_requests_total[5m]))", + "legendFormat": "{{status_code}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "pieType": "pie", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + } + } + }, + { + "id": 8, + "type": "stat", + "title": "App Target Uptime", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "expr": "up{job=\"app\"}", + "legendFormat": "app", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + } + }, + { + "id": 9, + "type": "timeseries", + "title": "System Info Collection p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(devops_info_system_collection_seconds_bucket[5m])))", + "legendFormat": "system info p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + } + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} diff --git a/ansible/roles/monitoring/tasks/main.yml b/ansible/roles/monitoring/tasks/main.yml index a30f1d9812..a0d3f2b674 100644 --- a/ansible/roles/monitoring/tasks/main.yml +++ b/ansible/roles/monitoring/tasks/main.yml @@ -12,6 +12,7 @@ - "{{ monitoring_project_dir }}" - "{{ monitoring_project_dir }}/loki" - "{{ monitoring_project_dir }}/promtail" + - "{{ monitoring_project_dir }}/prometheus" - "{{ monitoring_project_dir }}/grafana" - "{{ monitoring_project_dir }}/grafana/provisioning" - "{{ monitoring_project_dir }}/grafana/provisioning/datasources" @@ -36,7 +37,7 @@ group: root mode: "0600" - - name: Template monitoring stack files + - name: Template monitoring stack configuration files ansible.builtin.template: src: "{{ item.src }}" dest: "{{ monitoring_project_dir }}/{{ item.dest }}" @@ -47,10 +48,19 @@ - { src: 'docker-compose.yml.j2', dest: 'docker-compose.yml' } - { src: 'loki-config.yml.j2', dest: 'loki/config.yml' } - { src: 'promtail-config.yml.j2', dest: 'promtail/config.yml' } + - { src: 'prometheus-config.yml.j2', dest: 'prometheus/prometheus.yml' } - { src: 'grafana-datasource.yml.j2', dest: 'grafana/provisioning/datasources/loki.yml' } - { src: 'grafana-dashboard-provider.yml.j2', dest: 'grafana/provisioning/dashboards/dashboard-provider.yml' } - { src: 'grafana-dashboard.json.j2', dest: 'grafana/dashboards/lab07-logging.json' } + - name: Copy Lab08 Grafana metrics dashboard + ansible.builtin.copy: + src: lab08-metrics.json + dest: "{{ monitoring_project_dir }}/grafana/dashboards/lab08-metrics.json" + owner: root + group: root + mode: "0644" + - name: Deploy monitoring stack with Docker Compose v2 community.docker.docker_compose_v2: project_src: "{{ monitoring_project_dir }}" @@ -70,6 +80,16 @@ delay: 3 until: loki_ready.status == 200 + - name: Wait for Prometheus to become healthy + ansible.builtin.uri: + url: "http://127.0.0.1:{{ monitoring_prometheus_port }}/-/healthy" + method: GET + status_code: 200 + register: prometheus_ready + retries: 20 + delay: 3 + until: prometheus_ready.status == 200 + - name: Wait for Grafana API health endpoint ansible.builtin.uri: url: "http://127.0.0.1:{{ monitoring_grafana_port }}/api/health" @@ -93,14 +113,32 @@ delay: 3 until: monitoring_app_health.status == 200 - - name: Verify Loki data source was provisioned + - name: Wait for monitored application metrics endpoint ansible.builtin.uri: - url: "http://127.0.0.1:{{ monitoring_grafana_port }}/api/datasources/uid/{{ monitoring_grafana_datasource_uid }}" + url: "http://127.0.0.1:{{ monitoring_app_port }}/metrics" + method: GET + status_code: 200 + return_content: true + register: monitoring_app_metrics + retries: 20 + delay: 3 + until: + - monitoring_app_metrics.status == 200 + - "'http_requests_total' in monitoring_app_metrics.content" + + - name: Verify provisioned Grafana data sources + ansible.builtin.uri: + url: "http://127.0.0.1:{{ monitoring_grafana_port }}/api/datasources/uid/{{ item.uid }}" method: GET user: "{{ monitoring_grafana_admin_user }}" password: "{{ monitoring_grafana_admin_password }}" force_basic_auth: true status_code: 200 + loop: + - { name: '{{ monitoring_loki_datasource_name }}', uid: '{{ monitoring_loki_datasource_uid }}' } + - { name: '{{ monitoring_prometheus_datasource_name }}', uid: '{{ monitoring_prometheus_datasource_uid }}' } + loop_control: + label: "{{ item.name }}" rescue: - name: Monitoring deployment failure hint diff --git a/ansible/roles/monitoring/templates/docker-compose.yml.j2 b/ansible/roles/monitoring/templates/docker-compose.yml.j2 index 52f249fbf9..972a969a60 100644 --- a/ansible/roles/monitoring/templates/docker-compose.yml.j2 +++ b/ansible/roles/monitoring/templates/docker-compose.yml.j2 @@ -19,9 +19,9 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3100/ready"] - interval: 15s + interval: 10s timeout: 5s - retries: 10 + retries: 5 start_period: 20s deploy: resources: @@ -55,9 +55,9 @@ services: condition: service_healthy healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:9080/ready"] - interval: 15s + interval: 10s timeout: 5s - retries: 10 + retries: 5 start_period: 20s deploy: resources: @@ -77,7 +77,9 @@ services: GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER:-{{ monitoring_grafana_admin_user }}}" GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}" GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer GF_SECURITY_ALLOW_EMBEDDING: "false" + GF_METRICS_ENABLED: "true" ports: - "{{ monitoring_grafana_port }}:3000" volumes: @@ -95,9 +97,9 @@ services: condition: service_healthy healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3000/api/health"] - interval: 15s + interval: 10s timeout: 5s - retries: 10 + retries: 5 start_period: 30s deploy: resources: @@ -108,6 +110,48 @@ services: cpus: "{{ monitoring_resource_limits.grafana.reservations.cpus }}" memory: {{ monitoring_resource_limits.grafana.reservations.memory }} + prometheus: + image: prom/prometheus:{{ monitoring_prometheus_version }} + container_name: devops-prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time={{ monitoring_prometheus_retention_days }}d + - --storage.tsdb.retention.size={{ monitoring_prometheus_retention_size }} + - --web.enable-lifecycle + ports: + - "{{ monitoring_prometheus_port }}:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + networks: + - logging + labels: + logging: "promtail" + app: "devops-prometheus" + restart: unless-stopped + depends_on: + loki: + condition: service_healthy + grafana: + condition: service_healthy + {{ monitoring_app_service_name }}: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:9090/-/healthy"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + deploy: + resources: + limits: + cpus: "{{ monitoring_resource_limits.prometheus.limits.cpus }}" + memory: {{ monitoring_resource_limits.prometheus.limits.memory }} + reservations: + cpus: "{{ monitoring_resource_limits.prometheus.reservations.cpus }}" + memory: {{ monitoring_resource_limits.prometheus.reservations.memory }} + {{ monitoring_app_service_name }}: build: context: ./app-python @@ -128,9 +172,9 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:{{ monitoring_app_internal_port }}/health')"] - interval: 15s + interval: 10s timeout: 5s - retries: 10 + retries: 5 start_period: 20s deploy: resources: @@ -149,3 +193,4 @@ volumes: loki-data: grafana-data: promtail-data: + prometheus-data: diff --git a/ansible/roles/monitoring/templates/env.j2 b/ansible/roles/monitoring/templates/env.j2 index 6afa6c1271..c922f823ea 100644 --- a/ansible/roles/monitoring/templates/env.j2 +++ b/ansible/roles/monitoring/templates/env.j2 @@ -3,4 +3,5 @@ GRAFANA_ADMIN_PASSWORD={{ monitoring_grafana_admin_password }} GRAFANA_PORT={{ monitoring_grafana_port }} LOKI_PORT={{ monitoring_loki_port }} PROMTAIL_PORT={{ monitoring_promtail_port }} +PROMETHEUS_PORT={{ monitoring_prometheus_port }} APP_PORT={{ monitoring_app_port }} diff --git a/ansible/roles/monitoring/templates/grafana-datasource.yml.j2 b/ansible/roles/monitoring/templates/grafana-datasource.yml.j2 index f607bfadda..a19cd5ec5c 100644 --- a/ansible/roles/monitoring/templates/grafana-datasource.yml.j2 +++ b/ansible/roles/monitoring/templates/grafana-datasource.yml.j2 @@ -1,14 +1,16 @@ apiVersion: 1 deleteDatasources: - - name: {{ monitoring_grafana_datasource_name }} + - name: {{ monitoring_loki_datasource_name }} + orgId: 1 + - name: {{ monitoring_prometheus_datasource_name }} orgId: 1 prune: true datasources: - - name: {{ monitoring_grafana_datasource_name }} - uid: {{ monitoring_grafana_datasource_uid }} + - name: {{ monitoring_loki_datasource_name }} + uid: {{ monitoring_loki_datasource_uid }} type: loki access: proxy url: http://loki:3100 @@ -17,3 +19,15 @@ datasources: jsonData: maxLines: 1000 timeout: 60 + + - name: {{ monitoring_prometheus_datasource_name }} + uid: {{ monitoring_prometheus_datasource_uid }} + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: false + editable: false + jsonData: + httpMethod: POST + prometheusType: Prometheus + timeInterval: {{ monitoring_prometheus_scrape_interval }} diff --git a/ansible/roles/monitoring/templates/prometheus-config.yml.j2 b/ansible/roles/monitoring/templates/prometheus-config.yml.j2 new file mode 100644 index 0000000000..f3ffd4378e --- /dev/null +++ b/ansible/roles/monitoring/templates/prometheus-config.yml.j2 @@ -0,0 +1,16 @@ +global: + scrape_interval: {{ monitoring_prometheus_scrape_interval }} + evaluation_interval: {{ monitoring_prometheus_scrape_interval }} + +scrape_configs: +{% for target in monitoring_prometheus_targets %} + - job_name: {{ target.job | quote }} +{% if target.path is defined %} + metrics_path: {{ target.path | quote }} +{% endif %} + static_configs: + - targets: +{% for endpoint in target.targets %} + - {{ endpoint | quote }} +{% endfor %} +{% endfor %} diff --git a/app_python/app.py b/app_python/app.py index ddca2ada41..428b98833a 100644 --- a/app_python/app.py +++ b/app_python/app.py @@ -5,6 +5,7 @@ Endpoints: - GET / : service + system + runtime + request info - GET /health : health check (for probes/monitoring) +- GET /metrics : Prometheus metrics endpoint """ from __future__ import annotations @@ -19,7 +20,8 @@ from datetime import datetime, timezone from typing import Any, Dict -from flask import Flask, g, jsonify, request +from flask import Flask, Response, g, jsonify, request +from prometheus_client import CONTENT_TYPE_LATEST, Counter, Gauge, Histogram, generate_latest # ----------------------------------------------------------------------------- # App & Config @@ -39,6 +41,44 @@ START_TIME_UTC = datetime.now(timezone.utc) +# ----------------------------------------------------------------------------- +# Prometheus metrics +# ----------------------------------------------------------------------------- + +HTTP_REQUESTS_TOTAL = Counter( + "http_requests_total", + "Total HTTP requests processed by the service.", + ["method", "endpoint", "status_code"], +) + +HTTP_REQUEST_DURATION_SECONDS = Histogram( + "http_request_duration_seconds", + "HTTP request duration in seconds.", + ["method", "endpoint"], +) + +HTTP_REQUESTS_IN_PROGRESS = Gauge( + "http_requests_in_progress", + "HTTP requests currently being processed.", +) + +DEVOPS_INFO_ENDPOINT_CALLS_TOTAL = Counter( + "devops_info_endpoint_calls_total", + "Total endpoint calls for the DevOps info service.", + ["endpoint"], +) + +DEVOPS_INFO_SYSTEM_COLLECTION_SECONDS = Histogram( + "devops_info_system_collection_seconds", + "Time spent collecting system information.", +) + +DEVOPS_INFO_UPTIME_SECONDS = Gauge( + "devops_info_uptime_seconds", + "Current service uptime in seconds.", +) + + # ----------------------------------------------------------------------------- # Logging # ----------------------------------------------------------------------------- @@ -122,9 +162,31 @@ def get_client_ip() -> str: return request.remote_addr or "unknown" +def normalize_endpoint() -> str: + """ + Keep endpoint labels low-cardinality for Prometheus. + Uses the Flask route template when available and groups unknown paths. + """ + if request.url_rule and request.url_rule.rule: + return request.url_rule.rule + + if request.path == "/": + return "/" + + return "unmatched" + + @app.before_request def log_request_started() -> None: g.request_started_at = time.perf_counter() + g.normalized_endpoint = normalize_endpoint() + g.skip_http_metrics = request.path == "/metrics" + g.active_request_metric_registered = False + + if not g.skip_http_metrics: + HTTP_REQUESTS_IN_PROGRESS.inc() + g.active_request_metric_registered = True + logger.debug( "request_started", extra={ @@ -139,11 +201,36 @@ def log_request_started() -> None: ) +@app.teardown_request +def track_request_finished(_error: Exception | None) -> None: + if getattr(g, "active_request_metric_registered", False): + HTTP_REQUESTS_IN_PROGRESS.dec() + g.active_request_metric_registered = False + + @app.after_request def add_headers(response): - response.headers["Content-Type"] = "application/json; charset=utf-8" + endpoint = getattr(g, "normalized_endpoint", normalize_endpoint()) + duration_seconds = time.perf_counter() - getattr(g, "request_started_at", time.perf_counter()) + duration_ms = round(duration_seconds * 1000, 2) + + if not getattr(g, "skip_http_metrics", False): + HTTP_REQUESTS_TOTAL.labels( + method=request.method, + endpoint=endpoint, + status_code=str(response.status_code), + ).inc() + HTTP_REQUEST_DURATION_SECONDS.labels( + method=request.method, + endpoint=endpoint, + ).observe(duration_seconds) + DEVOPS_INFO_ENDPOINT_CALLS_TOTAL.labels(endpoint=endpoint).inc() + + DEVOPS_INFO_UPTIME_SECONDS.set(get_uptime()["seconds"]) + + if response.mimetype == "application/json": + response.headers["Content-Type"] = "application/json; charset=utf-8" - duration_ms = round((time.perf_counter() - getattr(g, "request_started_at", time.perf_counter())) * 1000, 2) logger.info( "request_completed", extra={ @@ -185,14 +272,18 @@ def get_uptime() -> Dict[str, Any]: def get_system_info() -> Dict[str, Any]: """Collect system information using Python standard library.""" - return { - "hostname": socket.gethostname(), - "platform": platform.system(), - "platform_version": platform.platform(), - "architecture": platform.machine(), - "cpu_count": os.cpu_count() or 0, - "python_version": platform.python_version(), - } + started_at = time.perf_counter() + try: + return { + "hostname": socket.gethostname(), + "platform": platform.system(), + "platform_version": platform.platform(), + "architecture": platform.machine(), + "cpu_count": os.cpu_count() or 0, + "python_version": platform.python_version(), + } + finally: + DEVOPS_INFO_SYSTEM_COLLECTION_SECONDS.observe(time.perf_counter() - started_at) @@ -200,6 +291,7 @@ def build_endpoints() -> list[Dict[str, str]]: return [ {"path": "/", "method": "GET", "description": "Service information"}, {"path": "/health", "method": "GET", "description": "Health check"}, + {"path": "/metrics", "method": "GET", "description": "Prometheus metrics"}, ] @@ -251,6 +343,13 @@ def health(): ), 200 +@app.get("/metrics") +def metrics() -> Response: + """Expose Prometheus metrics for scraping.""" + DEVOPS_INFO_UPTIME_SECONDS.set(get_uptime()["seconds"]) + return Response(generate_latest(), content_type=CONTENT_TYPE_LATEST) + + # ----------------------------------------------------------------------------- # Error Handlers # ----------------------------------------------------------------------------- diff --git a/app_python/requirements.txt b/app_python/requirements.txt index 78180a1ad1..46c776bf8d 100644 --- a/app_python/requirements.txt +++ b/app_python/requirements.txt @@ -1 +1,2 @@ -Flask==3.1.0 \ No newline at end of file +Flask==3.1.0 +prometheus-client==0.23.1 diff --git a/monitoring/.env.example b/monitoring/.env.example index 7b1de1cf95..dd8134271f 100644 --- a/monitoring/.env.example +++ b/monitoring/.env.example @@ -3,8 +3,9 @@ # Bash: cp .env.example .env GRAFANA_ADMIN_USER=admin -GRAFANA_ADMIN_PASSWORD=ChangeMe_Lab07! +GRAFANA_ADMIN_PASSWORD=ChangeMe_Lab08! GRAFANA_PORT=3000 LOKI_PORT=3100 PROMTAIL_PORT=9080 +PROMETHEUS_PORT=9090 APP_PORT=8000 diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 91209015d4..c32068b172 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -19,9 +19,9 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3100/ready"] - interval: 15s + interval: 10s timeout: 5s - retries: 10 + retries: 5 start_period: 20s deploy: resources: @@ -55,14 +55,14 @@ services: condition: service_healthy healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:9080/ready"] - interval: 15s + interval: 10s timeout: 5s - retries: 10 + retries: 5 start_period: 20s deploy: resources: limits: - cpus: "0.75" + cpus: "0.5" memory: 512M reservations: cpus: "0.10" @@ -79,6 +79,7 @@ services: GF_AUTH_ANONYMOUS_ENABLED: "false" GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer GF_SECURITY_ALLOW_EMBEDDING: "false" + GF_METRICS_ENABLED: "true" ports: - "${GRAFANA_PORT:-3000}:3000" volumes: @@ -96,10 +97,52 @@ services: condition: service_healthy healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3000/api/health"] - interval: 15s + interval: 10s timeout: 5s - retries: 10 + retries: 5 start_period: 30s + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + reservations: + cpus: "0.10" + memory: 128M + + prometheus: + image: prom/prometheus:v3.9.0 + container_name: devops-prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --storage.tsdb.retention.size=10GB + - --web.enable-lifecycle + ports: + - "${PROMETHEUS_PORT:-9090}:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + networks: + - logging + labels: + logging: "promtail" + app: "devops-prometheus" + restart: unless-stopped + depends_on: + loki: + condition: service_healthy + grafana: + condition: service_healthy + app-python: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:9090/-/healthy"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s deploy: resources: limits: @@ -113,7 +156,7 @@ services: build: context: ../app_python dockerfile: Dockerfile - image: devops-info-service:lab07 + image: devops-info-service:lab08 container_name: devops-python environment: HOST: "0.0.0.0" @@ -129,15 +172,15 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health')"] - interval: 15s + interval: 10s timeout: 5s - retries: 10 + retries: 5 start_period: 20s deploy: resources: limits: - cpus: "0.75" - memory: 512M + cpus: "0.5" + memory: 256M reservations: cpus: "0.10" memory: 128M @@ -150,3 +193,4 @@ volumes: loki-data: grafana-data: promtail-data: + prometheus-data: diff --git a/monitoring/docs/LAB08.md b/monitoring/docs/LAB08.md new file mode 100644 index 0000000000..2a1e0dfd2a --- /dev/null +++ b/monitoring/docs/LAB08.md @@ -0,0 +1,522 @@ +# Lab08 — Metrics & Monitoring with Prometheus + +## Summary + +This implementation extends the existing Lab07 observability stack with Prometheus-based metrics collection and a Grafana metrics dashboard. + +The solution is designed for a fully local, free workflow: +- Windows host +- WSL2 for Linux tooling +- optional Vagrant VM for a 100% local deployment target +- no external cloud services required + +The repository now contains: +- Prometheus instrumentation in the Python Flask application +- a `/metrics` endpoint compatible with Prometheus scraping +- a Prometheus service added to `monitoring/docker-compose.yml` +- Grafana provisioning for both Loki and Prometheus data sources +- a prebuilt Grafana metrics dashboard with 9 panels +- updated local validation instructions for Windows + WSL2 + Vagrant +- extended Ansible automation for the full Loki + Prometheus + Grafana stack + +## Architecture + +```text + +-----------------------+ + | Grafana | + | dashboards + Explore | + +-----------+-----------+ + ^ + | + +--------------+---------------+ + | | + | | + metrics | | logs + | | + | | ++---------------------+-----+ +---------+---------+ +| Python Flask App | | Promtail | +| /, /health, /metrics | | Docker SD + labels| ++------------+--------------+ +---------+---------+ + | | + | scrape | push + v v + +------+----------------+ +-------+--------+ + | Prometheus | | Loki | + | pull-based TSDB | | log storage | + +-----------------------+ +----------------+ +``` + +## Repository Structure + +```text +monitoring/ +├── .env.example +├── docker-compose.yml +├── docs/ +│ ├── LAB07.md +│ ├── LAB08.md +│ ├── LOCAL_VALIDATION_WINDOWS.md +│ └── screenshots/ +├── grafana/ +│ ├── dashboards/ +│ │ ├── lab07-logging.json +│ │ └── lab08-metrics.json +│ └── provisioning/ +│ ├── dashboards/ +│ │ └── dashboard-provider.yml +│ └── datasources/ +│ └── loki.yml +├── loki/ +│ └── config.yml +├── prometheus/ +│ └── prometheus.yml +└── promtail/ + └── config.yml +``` + +## Application Instrumentation + +### Metrics added to the Flask application + +#### 1. HTTP request counter + +Metric: + +```text +http_requests_total{method, endpoint, status_code} +``` + +Purpose: +- total request volume +- request rate calculations +- status code distribution +- error rate calculations + +#### 2. HTTP request duration histogram + +Metric: + +```text +http_request_duration_seconds{method, endpoint} +``` + +Purpose: +- latency measurements +- percentile calculations such as p95 +- heatmap visualizations for duration buckets + +Note: +- `status_code` is intentionally not used on the latency histogram to avoid unnecessary cardinality growth and to keep PromQL queries simpler + +#### 3. In-progress requests gauge + +Metric: + +```text +http_requests_in_progress +``` + +Purpose: +- concurrent request visibility +- simple saturation signal +- active request panel in Grafana + +#### 4. Application-specific business metrics + +Metrics: + +```text +devops_info_endpoint_calls_total{endpoint} +devops_info_system_collection_seconds +devops_info_uptime_seconds +``` + +Purpose: +- endpoint usage tracking +- system information collection cost tracking +- current service uptime for dashboards and troubleshooting + +### Instrumentation design choices + +- request metrics are collected in Flask hooks (`before_request`, `after_request`, `teardown_request`) +- endpoint labels are normalized to avoid high-cardinality labels +- unknown routes are grouped as `unmatched` +- `/metrics` scrape traffic is excluded from request-rate business metrics to avoid self-scrape noise in dashboards +- `/metrics` keeps the Prometheus content type and is not overwritten by the JSON response middleware +- the in-progress gauge is decremented in `teardown_request` to avoid leaks in error scenarios + +## Prometheus Configuration + +File: + +```text +monitoring/prometheus/prometheus.yml +``` + +### Scrape settings + +- scrape interval: `15s` +- evaluation interval: `15s` + +### Scrape targets + +1. `prometheus` + - target: `localhost:9090` +2. `app` + - target: `app-python:8000` + - path: `/metrics` +3. `loki` + - target: `loki:3100` +4. `grafana` + - target: `grafana:3000` + +### Retention policy + +Prometheus retention is configured via container flags in `docker-compose.yml`: + +```text +--storage.tsdb.retention.time=15d +--storage.tsdb.retention.size=10GB +``` + +Why this matters: +- avoids unbounded disk usage +- keeps local storage predictable on a laptop or VM +- is fully compatible with a free local deployment model + +## Grafana Dashboard Walkthrough + +Dashboard file: + +```text +monitoring/grafana/dashboards/lab08-metrics.json +``` + +The dashboard includes 9 panels. + +### 1. Request Rate by Endpoint + +Query: + +```promql +sum by (endpoint) (rate(http_requests_total{endpoint!="/metrics"}[5m])) +``` + +Purpose: +- visualizes request throughput +- supports the **R** in RED + +### 2. Error Rate (5xx) + +Query: + +```promql +sum(rate(http_requests_total{status_code=~"5.."}[5m])) +``` + +Purpose: +- shows server-side errors per second +- supports the **E** in RED + +### 3. Request Duration p95 by Endpoint + +Query: + +```promql +histogram_quantile(0.95, sum by (le, endpoint) (rate(http_request_duration_seconds_bucket{endpoint!="/metrics"}[5m]))) +``` + +Purpose: +- tracks latency percentiles +- supports the **D** in RED + +### 4. Active Requests + +Query: + +```promql +sum(http_requests_in_progress) +``` + +Purpose: +- highlights current concurrency pressure + +### 5. Application Uptime + +Query: + +```promql +max(devops_info_uptime_seconds) +``` + +Purpose: +- confirms the service is alive and progressing normally + +### 6. Request Duration Heatmap + +Query: + +```promql +sum by (le) (rate(http_request_duration_seconds_bucket{endpoint!="/metrics"}[5m])) +``` + +Purpose: +- visualizes latency bucket distribution over time + +### 7. Status Code Distribution + +Query: + +```promql +sum by (status_code) (rate(http_requests_total[5m])) +``` + +Purpose: +- shows how traffic is split across response classes and codes + +### 8. App Target Uptime + +Query: + +```promql +up{job="app"} +``` + +Purpose: +- shows whether Prometheus is successfully scraping the application + +### 9. System Info Collection p95 + +Query: + +```promql +histogram_quantile(0.95, sum by (le) (rate(devops_info_system_collection_seconds_bucket[5m]))) +``` + +Purpose: +- tracks the internal cost of the service-specific system information collection function + +## PromQL Examples + +### RED method queries + +```promql +sum by (endpoint) (rate(http_requests_total{endpoint!="/metrics"}[5m])) +sum(rate(http_requests_total{status_code=~"5.."}[5m])) +histogram_quantile(0.95, sum by (le, endpoint) (rate(http_request_duration_seconds_bucket{endpoint!="/metrics"}[5m]))) +``` + +### Additional useful queries + +```promql +sum by (status_code) (rate(http_requests_total[5m])) +sum(http_requests_in_progress) +up{job="app"} +max(devops_info_uptime_seconds) +sum by (job) (up) +histogram_quantile(0.95, sum by (le) (rate(devops_info_system_collection_seconds_bucket[5m]))) +``` + +## Production Setup + +### Health checks + +Health checks are configured for: +- Loki +- Promtail +- Grafana +- Prometheus +- Python application + +### Resource limits + +Configured Docker Compose limits: + +| Service | CPU | Memory | +|--------|-----|--------| +| Loki | 1.0 | 1G | +| Promtail | 0.5 | 512M | +| Grafana | 0.5 | 512M | +| Prometheus | 1.0 | 1G | +| App | 0.5 | 256M | + +### Persistence + +Named volumes: +- `loki-data` +- `promtail-data` +- `grafana-data` +- `prometheus-data` + +### Security and operational choices + +- Grafana anonymous access is disabled +- Grafana credentials are externalized via `.env` +- Grafana metrics are explicitly enabled for Prometheus scraping +- the app health check uses Python stdlib instead of `curl` to keep the slim image lightweight + +## Local Run Commands + +### Docker Compose from WSL2 + +```bash +cd monitoring +cp .env.example .env +# edit GRAFANA_ADMIN_PASSWORD in .env + +docker compose up -d --build +docker compose ps +``` + +### Generate traffic + +```bash +for i in {1..25}; do curl -s http://127.0.0.1:8000/ > /dev/null; done +for i in {1..25}; do curl -s http://127.0.0.1:8000/health > /dev/null; done +curl -s http://127.0.0.1:8000/does-not-exist > /dev/null +curl -s http://127.0.0.1:8000/metrics | head -40 +``` + +### Validate Prometheus + +```bash +curl http://127.0.0.1:9090/-/healthy +curl http://127.0.0.1:9090/api/v1/targets | python -m json.tool +``` + +Open in browser: +- `http://127.0.0.1:9090/targets` +- `http://127.0.0.1:3000` + +### Useful Grafana checks + +- Confirm both data sources are provisioned: Loki and Prometheus +- Open dashboard: `Lab08 - Prometheus Metrics Overview` +- Run ad hoc PromQL query: `up` + +## Testing Results + +### Static validation completed in the sandbox + +The following checks were completed while preparing this repository update: +- Python syntax validation for the Flask application +- JSON validation for the Grafana dashboard file +- YAML validation for Docker Compose, Prometheus, Loki, and Promtail configuration files +- local application runtime validation for `/`, `/health`, `/metrics`, and request counters + +Captured application-side evidence is stored in: + +```text +monitoring/docs/evidence/ +``` + +Files included there: +- `app-root.json` +- `app-health.json` +- `app-metrics-sample.txt` +- `app-metrics-headers.txt` + +### Full stack runtime validation to run locally + +Because the sandbox used to prepare this patch does not provide a Docker daemon, UI screenshots and multi-container runtime proofs must be captured locally with the provided commands. + +Recommended screenshots to capture locally: +- all `lab08*.png` files in `monitoring/docs/screenshots` + + +## Metrics vs Logs + +### Use metrics when you need +- rates and trends over time +- low-cost aggregation +- alert thresholds +- latency and saturation views + +### Use logs when you need +- exact event details +- request-specific context +- raw error messages and stack traces +- forensic troubleshooting + +### Combined observability model in this repository + +- **Loki + Promtail** answer: *what happened?* +- **Prometheus** answers: *how much, how often, how fast?* +- **Grafana** provides one UI for both views + +## Challenges and Solutions + +### 1. `/metrics` vs JSON middleware + +Problem: +- the existing `after_request` hook forced `application/json` on every response + +Solution: +- the hook now preserves the Prometheus metrics content type and only rewrites JSON responses + +### 2. Label cardinality control + +Problem: +- raw request paths can create unbounded label values + +Solution: +- metrics use normalized endpoints and group unknown routes as `unmatched` + +### 3. In-progress gauge correctness + +Problem: +- a gauge can leak if it is incremented but not decremented during exceptions + +Solution: +- the decrement is handled in `teardown_request` + +### 4. Windows + WSL2 + Vagrant networking + +Problem: +- forwarded ports can behave differently in Windows and WSL2 + +Solution: +- the local validation guide includes both localhost and host-IP guidance + +### 5. Free local deployment requirement + +Problem: +- the lab should not depend on paid services or third-party cloud platforms + +Solution: +- the full stack runs locally with Docker Compose and can also run on the Vagrant VM via Ansible + +## Bonus — Ansible Automation + +The existing `ansible/roles/monitoring` role was extended to cover Lab08 as well. + +Implemented bonus scope: +- Prometheus variables added to role defaults +- Prometheus config generated from a Jinja2 template +- Docker Compose template updated to include Prometheus +- Grafana provisioning now includes both Loki and Prometheus data sources +- the Lab08 dashboard JSON is rendered by Ansible automatically +- the deployment task verifies Prometheus health and both data sources + +Playbook: + +```bash +cd ansible +ansible-galaxy collection install -r requirements.yml +ansible-playbook -i inventory/hosts.ini playbooks/deploy-monitoring.yml +``` + +## Evidence Checklist + +After you run the stack locally, verify the following: + +- `/metrics` endpoint returns Prometheus-formatted metrics +- Prometheus `/targets` shows all targets as `UP` +- Grafana has both Loki and Prometheus data sources +- the Lab08 dashboard shows live data in all panels +- `docker compose ps` shows healthy containers +- dashboards survive `docker compose down` / `docker compose up -d` + +All these checklist you can see as a screenshots in: +- all `lab08*.png` files in `monitoring/docs/screenshots` diff --git a/monitoring/docs/LOCAL_VALIDATION_WINDOWS.md b/monitoring/docs/LOCAL_VALIDATION_WINDOWS.md index 7055bf9974..f66c637525 100644 --- a/monitoring/docs/LOCAL_VALIDATION_WINDOWS.md +++ b/monitoring/docs/LOCAL_VALIDATION_WINDOWS.md @@ -1,80 +1,119 @@ -# Lab07 Local Validation on Windows 11 (WSL2 + Vagrant-friendly) - -This guide is tailored for the same environment used in earlier labs: -- Windows 11 host -- WSL2 Ubuntu for Linux commands -- VirtualBox + Vagrant for the free local VM option -- Russia / unstable access to Docker registries is possible, so retries are expected - -## Option A — Run the stack directly from WSL (if you already have a Linux Docker engine) - -1. Copy the example environment file: - ```bash - cd monitoring - cp .env.example .env - ``` -2. Edit `.env` and set a real Grafana password. -3. Start the stack: - ```bash - docker compose up -d --build - docker compose ps - ``` -4. Verify endpoints: - ```bash - curl http://127.0.0.1:3100/ready - curl http://127.0.0.1:9080/targets - curl http://127.0.0.1:8000/health - ``` -5. Open Grafana in a browser: `http://127.0.0.1:3000` - - user: `admin` - - password: value from `.env` - -## Option B — 100% free path using the Vagrant VM and Ansible bonus automation - -1. Reload Vagrant to apply the new forwarded ports: - ```powershell - vagrant reload - vagrant status - vagrant port - ``` -2. Make sure the VM is reachable from WSL. If forwarded ports do not work via `127.0.0.1` inside WSL, use the current Windows host IP from the `vEthernet (WSL...)` adapter. -3. From WSL, activate your Python environment and run: - ```bash - source ~/venvs/devops-lab/bin/activate - export ANSIBLE_CONFIG=/home//work/ansible/ansible.cfg - cd /path/to/repo/ansible - ansible-galaxy collection install -r requirements.yml - ansible-playbook -i inventory/hosts.ini playbooks/deploy-monitoring.yml - ``` -4. From Windows open: - - Grafana: `http://127.0.0.1:3000` - - Loki ready endpoint: `http://127.0.0.1:3100/ready` - - App health: `http://127.0.0.1:8000/health` -5. From WSL, if `127.0.0.1` forwarding does not work, use the current Windows host IP instead of localhost. - -## Generate example traffic - -### Bash / WSL +# Local Validation on Windows 11 (WSL2 + Vagrant-friendly) + +This guide is tailored for a fully local and free setup: +- Windows host +- WSL2 for Linux commands +- VS Code connected to WSL2 +- optional Vagrant VM for a dedicated Linux target +- no external cloud services required + +## Ports used by the observability stack + +- App: `8000` +- Grafana: `3000` +- Loki: `3100` +- Promtail: `9080` +- Prometheus: `9090` + +## Option A — Run directly from WSL2 + +### 1. Prepare environment file + +```bash +cd monitoring +cp .env.example .env +``` + +Set a real Grafana password in `.env`. + +### 2. Start the stack + +```bash +docker compose up -d --build +docker compose ps +``` + +### 3. Verify endpoints + +```bash +curl http://127.0.0.1:8000/health +curl http://127.0.0.1:8000/metrics | head -40 +curl http://127.0.0.1:3100/ready +curl http://127.0.0.1:9080/targets +curl http://127.0.0.1:9090/-/healthy +``` + +### 4. Open browser pages + +- Grafana: `http://127.0.0.1:3000` +- Prometheus targets: `http://127.0.0.1:9090/targets` +- Prometheus graph: `http://127.0.0.1:9090/graph` + +### 5. Generate traffic + ```bash -for i in {1..20}; do curl -s http://127.0.0.1:8000/ > /dev/null; done -for i in {1..20}; do curl -s http://127.0.0.1:8000/health > /dev/null; done +for i in {1..30}; do curl -s http://127.0.0.1:8000/ > /dev/null; done +for i in {1..30}; do curl -s http://127.0.0.1:8000/health > /dev/null; done curl -s http://127.0.0.1:8000/does-not-exist > /dev/null ``` -### PowerShell +### 6. Prometheus queries to test + +```promql +up +sum by (endpoint) (rate(http_requests_total{endpoint!="/metrics"}[5m])) +sum(rate(http_requests_total{status_code=~"5.."}[5m])) +histogram_quantile(0.95, sum by (le, endpoint) (rate(http_request_duration_seconds_bucket{endpoint!="/metrics"}[5m]))) +sum by (status_code) (rate(http_requests_total[5m])) +``` + +### 7. Grafana checks + +- log in with the credentials from `.env` +- confirm both data sources exist: Loki and Prometheus +- open the `Lab08 - Prometheus Metrics Overview` dashboard +- verify the panels update after traffic generation + +## Option B — Run on the Vagrant VM + +### 1. Reload Vagrant to apply port forwarding + ```powershell -1..20 | ForEach-Object { Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/ | Out-Null } -1..20 | ForEach-Object { Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/health | Out-Null } -try { Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/does-not-exist | Out-Null } catch {} +vagrant reload +vagrant status +vagrant port ``` -## Useful LogQL checks +### 2. Run Ansible deployment from WSL2 -```logql -{job="docker"} -{app="devops-python"} -{app="devops-python"} | json | level="INFO" -{app="devops-python"} | json | level="ERROR" -sum by (app) (rate({app=~"devops-.*"}[1m])) -sum by (level) (count_over_time({app=~"devops-.*"} | json [5m])) +```bash +cd ansible +ansible-galaxy collection install -r requirements.yml +ansible-playbook -i inventory/hosts.ini playbooks/deploy-monitoring.yml ``` + +### 3. Verify from the Windows host or WSL2 + +```bash +curl http://127.0.0.1:8000/health +curl http://127.0.0.1:9090/-/healthy +``` + +If WSL2 cannot reach forwarded ports through `127.0.0.1`, use the current Windows host IP. + +## PowerShell traffic generation + +```powershell +1..30 | ForEach-Object { Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/ | Out-Null } +1..30 | ForEach-Object { Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/health | Out-Null } +try { Invoke-WebRequest -UseBasicParsing http://127.0.0.1:8000/does-not-exist | Out-Null } catch {} +``` + +## What to capture for proof + +- `/metrics` endpoint output in the browser or terminal +- Prometheus `/targets` page with all targets `UP` +- Prometheus query page with `up` +- Grafana Prometheus data source test +- Grafana dashboard with all panels populated +- `docker compose ps` showing healthy services diff --git a/monitoring/docs/evidence/README.md b/monitoring/docs/evidence/README.md new file mode 100644 index 0000000000..96a3a3a14e --- /dev/null +++ b/monitoring/docs/evidence/README.md @@ -0,0 +1,11 @@ +This directory contains lightweight validation artifacts captured from the sandbox for the instrumented Flask application itself. + +Included files: +- `app-root.json` — sample response from `GET /` +- `app-health.json` — sample response from `GET /health` +- `app-metrics-sample.txt` — sample Prometheus metrics output from `GET /metrics` +- `app-metrics-headers.txt` — response headers for `/metrics` confirming the Prometheus content type + +These files validate the application-side instrumentation. + +Full multi-container validation for Prometheus, Grafana, and Loki must still be executed locally because the sandbox does not provide a Docker daemon. diff --git a/monitoring/docs/evidence/app-health.json b/monitoring/docs/evidence/app-health.json new file mode 100644 index 0000000000..09e5ff36e3 --- /dev/null +++ b/monitoring/docs/evidence/app-health.json @@ -0,0 +1,5 @@ +{ + "status": "healthy", + "timestamp": "2026-03-19T13:05:23.900Z", + "uptime_seconds": 9 +} diff --git a/monitoring/docs/evidence/app-metrics-headers.txt b/monitoring/docs/evidence/app-metrics-headers.txt new file mode 100644 index 0000000000..a9447d71ae --- /dev/null +++ b/monitoring/docs/evidence/app-metrics-headers.txt @@ -0,0 +1,127 @@ +HTTP/1.1 200 OK +Server: Werkzeug/3.1.6 Python/3.13.5 +Date: Thu, 19 Mar 2026 13:05:16 GMT +Content-Type: text/plain; version=1.0.0; charset=utf-8 +Content-Length: 9658 +Connection: close + +# HELP python_gc_objects_collected_total Objects collected during gc +# TYPE python_gc_objects_collected_total counter +python_gc_objects_collected_total{generation="0"} 523.0 +python_gc_objects_collected_total{generation="1"} 0.0 +python_gc_objects_collected_total{generation="2"} 0.0 +# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC +# TYPE python_gc_objects_uncollectable_total counter +python_gc_objects_uncollectable_total{generation="0"} 0.0 +python_gc_objects_uncollectable_total{generation="1"} 0.0 +python_gc_objects_uncollectable_total{generation="2"} 0.0 +# HELP python_gc_collections_total Number of times this generation was collected +# TYPE python_gc_collections_total counter +python_gc_collections_total{generation="0"} 26.0 +python_gc_collections_total{generation="1"} 2.0 +python_gc_collections_total{generation="2"} 0.0 +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="13",patchlevel="5",version="3.13.5"} 1.0 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 1.35983104e+08 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 5.4464512e+07 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.77392551375e+09 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 0.26 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 6.0 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP http_requests_total Total HTTP requests processed by the service. +# TYPE http_requests_total counter +http_requests_total{endpoint="/",method="GET",status_code="200"} 1.0 +http_requests_total{endpoint="/health",method="GET",status_code="200"} 1.0 +http_requests_total{endpoint="unmatched",method="GET",status_code="404"} 1.0 +# HELP http_requests_created Total HTTP requests processed by the service. +# TYPE http_requests_created gauge +http_requests_created{endpoint="/",method="GET",status_code="200"} 1.773925515830881e+09 +http_requests_created{endpoint="/health",method="GET",status_code="200"} 1.7739255158618011e+09 +http_requests_created{endpoint="unmatched",method="GET",status_code="404"} 1.773925515895349e+09 +# HELP http_request_duration_seconds HTTP request duration in seconds. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{endpoint="/",le="0.005",method="GET"} 0.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.01",method="GET"} 0.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.025",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.05",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.075",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.1",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.25",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.75",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="1.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="2.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="5.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="7.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="10.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="+Inf",method="GET"} 1.0 +http_request_duration_seconds_count{endpoint="/",method="GET"} 1.0 +http_request_duration_seconds_sum{endpoint="/",method="GET"} 0.011629433000052813 +http_request_duration_seconds_bucket{endpoint="/health",le="0.005",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.01",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.025",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.05",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.075",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.1",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.25",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.75",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="1.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="2.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="5.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="7.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="10.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="+Inf",method="GET"} 1.0 +http_request_duration_seconds_count{endpoint="/health",method="GET"} 1.0 +http_request_duration_seconds_sum{endpoint="/health",method="GET"} 0.00015099899997039756 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.005",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.01",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.025",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.05",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.075",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.1",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.25",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.75",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="1.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="2.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="5.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="7.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="10.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="+Inf",method="GET"} 1.0 +http_request_duration_seconds_count{endpoint="unmatched",method="GET"} 1.0 +http_request_duration_seconds_sum{endpoint="unmatched",method="GET"} 0.00043901000003643276 +# HELP http_request_duration_seconds_created HTTP request duration in seconds. +# TYPE http_request_duration_seconds_created gauge +http_request_duration_seconds_created{endpoint="/",method="GET"} 1.773925515830906e+09 +http_request_duration_seconds_created{endpoint="/health",method="GET"} 1.7739255158618293e+09 +http_request_duration_seconds_created{endpoint="unmatched",method="GET"} 1.773925515895371e+09 +# HELP http_requests_in_progress HTTP requests currently being processed. +# TYPE http_requests_in_progress gauge +http_requests_in_progress 0.0 +# HELP devops_info_endpoint_calls_total Total endpoint calls for the DevOps info service. +# TYPE devops_info_endpoint_calls_total counter +devops_info_endpoint_calls_total{endpoint="/"} 1.0 +devops_info_endpoint_calls_total{endpoint="/health"} 1.0 +devops_info_endpoint_calls_total{endpoint="unmatched"} 1.0 +# HELP devops_info_endpoint_calls_created Total endpoint calls for the DevOps info service. +# TYPE devops_info_endpoint_calls_created gauge +devops_info_endpoint_calls_created{endpoint="/"} 1.7739255158309686e+09 +devops_info_endpoint_calls_created{endpoint="/health"} 1.7739255158618608e+09 +devops_info_endpoint_calls_created{endpoint="unmatched"} 1.773925515895407e+09 +# HELP devops_info_system_collection_seconds Time spent collecting system information. +# TYPE devops_info_system_collection_seconds histogram +devops_info_system_collection_seconds_bucket{le="0.005"} 0.0 diff --git a/monitoring/docs/evidence/app-metrics-sample.txt b/monitoring/docs/evidence/app-metrics-sample.txt new file mode 100644 index 0000000000..5b6436e2cf --- /dev/null +++ b/monitoring/docs/evidence/app-metrics-sample.txt @@ -0,0 +1,142 @@ +# HELP python_gc_objects_collected_total Objects collected during gc +# TYPE python_gc_objects_collected_total counter +python_gc_objects_collected_total{generation="0"} 523.0 +python_gc_objects_collected_total{generation="1"} 0.0 +python_gc_objects_collected_total{generation="2"} 0.0 +# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC +# TYPE python_gc_objects_uncollectable_total counter +python_gc_objects_uncollectable_total{generation="0"} 0.0 +python_gc_objects_uncollectable_total{generation="1"} 0.0 +python_gc_objects_uncollectable_total{generation="2"} 0.0 +# HELP python_gc_collections_total Number of times this generation was collected +# TYPE python_gc_collections_total counter +python_gc_collections_total{generation="0"} 26.0 +python_gc_collections_total{generation="1"} 2.0 +python_gc_collections_total{generation="2"} 0.0 +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="13",patchlevel="5",version="3.13.5"} 1.0 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 1.35983104e+08 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 5.4464512e+07 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.77392551375e+09 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 0.26 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 6.0 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP http_requests_total Total HTTP requests processed by the service. +# TYPE http_requests_total counter +http_requests_total{endpoint="/",method="GET",status_code="200"} 1.0 +http_requests_total{endpoint="/health",method="GET",status_code="200"} 1.0 +http_requests_total{endpoint="unmatched",method="GET",status_code="404"} 1.0 +# HELP http_requests_created Total HTTP requests processed by the service. +# TYPE http_requests_created gauge +http_requests_created{endpoint="/",method="GET",status_code="200"} 1.773925515830881e+09 +http_requests_created{endpoint="/health",method="GET",status_code="200"} 1.7739255158618011e+09 +http_requests_created{endpoint="unmatched",method="GET",status_code="404"} 1.773925515895349e+09 +# HELP http_request_duration_seconds HTTP request duration in seconds. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{endpoint="/",le="0.005",method="GET"} 0.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.01",method="GET"} 0.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.025",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.05",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.075",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.1",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.25",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="0.75",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="1.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="2.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="5.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="7.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="10.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/",le="+Inf",method="GET"} 1.0 +http_request_duration_seconds_count{endpoint="/",method="GET"} 1.0 +http_request_duration_seconds_sum{endpoint="/",method="GET"} 0.011629433000052813 +http_request_duration_seconds_bucket{endpoint="/health",le="0.005",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.01",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.025",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.05",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.075",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.1",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.25",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="0.75",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="1.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="2.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="5.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="7.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="10.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="/health",le="+Inf",method="GET"} 1.0 +http_request_duration_seconds_count{endpoint="/health",method="GET"} 1.0 +http_request_duration_seconds_sum{endpoint="/health",method="GET"} 0.00015099899997039756 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.005",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.01",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.025",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.05",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.075",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.1",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.25",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="0.75",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="1.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="2.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="5.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="7.5",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="10.0",method="GET"} 1.0 +http_request_duration_seconds_bucket{endpoint="unmatched",le="+Inf",method="GET"} 1.0 +http_request_duration_seconds_count{endpoint="unmatched",method="GET"} 1.0 +http_request_duration_seconds_sum{endpoint="unmatched",method="GET"} 0.00043901000003643276 +# HELP http_request_duration_seconds_created HTTP request duration in seconds. +# TYPE http_request_duration_seconds_created gauge +http_request_duration_seconds_created{endpoint="/",method="GET"} 1.773925515830906e+09 +http_request_duration_seconds_created{endpoint="/health",method="GET"} 1.7739255158618293e+09 +http_request_duration_seconds_created{endpoint="unmatched",method="GET"} 1.773925515895371e+09 +# HELP http_requests_in_progress HTTP requests currently being processed. +# TYPE http_requests_in_progress gauge +http_requests_in_progress 0.0 +# HELP devops_info_endpoint_calls_total Total endpoint calls for the DevOps info service. +# TYPE devops_info_endpoint_calls_total counter +devops_info_endpoint_calls_total{endpoint="/"} 1.0 +devops_info_endpoint_calls_total{endpoint="/health"} 1.0 +devops_info_endpoint_calls_total{endpoint="unmatched"} 1.0 +# HELP devops_info_endpoint_calls_created Total endpoint calls for the DevOps info service. +# TYPE devops_info_endpoint_calls_created gauge +devops_info_endpoint_calls_created{endpoint="/"} 1.7739255158309686e+09 +devops_info_endpoint_calls_created{endpoint="/health"} 1.7739255158618608e+09 +devops_info_endpoint_calls_created{endpoint="unmatched"} 1.773925515895407e+09 +# HELP devops_info_system_collection_seconds Time spent collecting system information. +# TYPE devops_info_system_collection_seconds histogram +devops_info_system_collection_seconds_bucket{le="0.005"} 0.0 +devops_info_system_collection_seconds_bucket{le="0.01"} 0.0 +devops_info_system_collection_seconds_bucket{le="0.025"} 1.0 +devops_info_system_collection_seconds_bucket{le="0.05"} 1.0 +devops_info_system_collection_seconds_bucket{le="0.075"} 1.0 +devops_info_system_collection_seconds_bucket{le="0.1"} 1.0 +devops_info_system_collection_seconds_bucket{le="0.25"} 1.0 +devops_info_system_collection_seconds_bucket{le="0.5"} 1.0 +devops_info_system_collection_seconds_bucket{le="0.75"} 1.0 +devops_info_system_collection_seconds_bucket{le="1.0"} 1.0 +devops_info_system_collection_seconds_bucket{le="2.5"} 1.0 +devops_info_system_collection_seconds_bucket{le="5.0"} 1.0 +devops_info_system_collection_seconds_bucket{le="7.5"} 1.0 +devops_info_system_collection_seconds_bucket{le="10.0"} 1.0 +devops_info_system_collection_seconds_bucket{le="+Inf"} 1.0 +devops_info_system_collection_seconds_count 1.0 +devops_info_system_collection_seconds_sum 0.011327940000001036 +# HELP devops_info_system_collection_seconds_created Time spent collecting system information. +# TYPE devops_info_system_collection_seconds_created gauge +devops_info_system_collection_seconds_created 1.7739255140238588e+09 +# HELP devops_info_uptime_seconds Current service uptime in seconds. +# TYPE devops_info_uptime_seconds gauge +devops_info_uptime_seconds 1.0 diff --git a/monitoring/docs/evidence/app-root.json b/monitoring/docs/evidence/app-root.json new file mode 100644 index 0000000000..005e35267b --- /dev/null +++ b/monitoring/docs/evidence/app-root.json @@ -0,0 +1,45 @@ +{ + "endpoints": [ + { + "description": "Service information", + "method": "GET", + "path": "/" + }, + { + "description": "Health check", + "method": "GET", + "path": "/health" + }, + { + "description": "Prometheus metrics", + "method": "GET", + "path": "/metrics" + } + ], + "request": { + "client_ip": "127.0.0.1", + "method": "GET", + "path": "/", + "user_agent": "curl/8.10.1" + }, + "runtime": { + "current_time": "2026-03-19T13:05:22.568Z", + "timezone": "UTC", + "uptime_human": "0 hours, 0 minutes", + "uptime_seconds": 8 + }, + "service": { + "description": "DevOps course info service", + "framework": "Flask", + "name": "devops-info-service", + "version": "1.1.0" + }, + "system": { + "architecture": "x86_64", + "cpu_count": 56, + "hostname": "17ceccfebd47", + "platform": "Linux", + "platform_version": "Linux-4.4.0-x86_64-with-glibc2.41", + "python_version": "3.13.5" + } +} diff --git a/monitoring/docs/screenshots/README.md b/monitoring/docs/screenshots/README.md index 25d39fa7a3..8b6572b859 100644 --- a/monitoring/docs/screenshots/README.md +++ b/monitoring/docs/screenshots/README.md @@ -1,6 +1,10 @@ Store your local proof screenshots here after you run the stack: -- grafana-explore.png +- metrics-endpoint.png +- prometheus-targets.png +- prometheus-query-up.png +- grafana-prometheus-datasource.png +- grafana-lab08-dashboard.png +- docker-compose-ps-healthy.png +- grafana-explore-app-logs.png - grafana-dashboard.png -- grafana-login.png -- app-json-logs.png diff --git a/monitoring/docs/screenshots/lab08-datasources.png b/monitoring/docs/screenshots/lab08-datasources.png new file mode 100644 index 0000000000..c607a53a92 Binary files /dev/null and b/monitoring/docs/screenshots/lab08-datasources.png differ diff --git a/monitoring/docs/screenshots/lab08-devops_info_uptime_seconds.png b/monitoring/docs/screenshots/lab08-devops_info_uptime_seconds.png new file mode 100644 index 0000000000..eaf983b8a8 Binary files /dev/null and b/monitoring/docs/screenshots/lab08-devops_info_uptime_seconds.png differ diff --git a/monitoring/docs/screenshots/lab08-docker-compose-ps.png b/monitoring/docs/screenshots/lab08-docker-compose-ps.png new file mode 100644 index 0000000000..8e518c34cd Binary files /dev/null and b/monitoring/docs/screenshots/lab08-docker-compose-ps.png differ diff --git a/monitoring/docs/screenshots/lab08-grafana-dasshboards.png b/monitoring/docs/screenshots/lab08-grafana-dasshboards.png new file mode 100644 index 0000000000..30740e612b Binary files /dev/null and b/monitoring/docs/screenshots/lab08-grafana-dasshboards.png differ diff --git a/monitoring/docs/screenshots/lab08-grafana-logs.png b/monitoring/docs/screenshots/lab08-grafana-logs.png new file mode 100644 index 0000000000..423709452f Binary files /dev/null and b/monitoring/docs/screenshots/lab08-grafana-logs.png differ diff --git a/monitoring/docs/screenshots/lab08-http_requests_total.png b/monitoring/docs/screenshots/lab08-http_requests_total.png new file mode 100644 index 0000000000..9771656300 Binary files /dev/null and b/monitoring/docs/screenshots/lab08-http_requests_total.png differ diff --git a/monitoring/docs/screenshots/lab08-prometheus-targets-up.png b/monitoring/docs/screenshots/lab08-prometheus-targets-up.png new file mode 100644 index 0000000000..88ad4c6365 Binary files /dev/null and b/monitoring/docs/screenshots/lab08-prometheus-targets-up.png differ diff --git a/monitoring/docs/screenshots/lab08-rate(http_requests_total[5m])-query.png b/monitoring/docs/screenshots/lab08-rate(http_requests_total[5m])-query.png new file mode 100644 index 0000000000..1ecfc5ad6e Binary files /dev/null and b/monitoring/docs/screenshots/lab08-rate(http_requests_total[5m])-query.png differ diff --git a/monitoring/docs/screenshots/lab08-up.png b/monitoring/docs/screenshots/lab08-up.png new file mode 100644 index 0000000000..8e1ca8d3fb Binary files /dev/null and b/monitoring/docs/screenshots/lab08-up.png differ diff --git a/monitoring/grafana/dashboards/lab08-metrics.json b/monitoring/grafana/dashboards/lab08-metrics.json new file mode 100644 index 0000000000..b05c0fa86a --- /dev/null +++ b/monitoring/grafana/dashboards/lab08-metrics.json @@ -0,0 +1,454 @@ +{ + "id": null, + "uid": "lab08-prometheus-metrics", + "title": "Lab08 - Prometheus Metrics Overview", + "tags": [ + "lab08", + "prometheus", + "metrics", + "observability" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Request Rate by Endpoint", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (endpoint) (rate(http_requests_total{endpoint!=\"/metrics\"}[5m]))", + "legendFormat": "{{endpoint}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Error Rate (5xx)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(http_requests_total{status_code=~\"5..\"}[5m]))", + "legendFormat": "5xx errors/sec", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Request Duration p95 by Endpoint", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, endpoint) (rate(http_request_duration_seconds_bucket{endpoint!=\"/metrics\"}[5m])))", + "legendFormat": "{{endpoint}} p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Active Requests", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(http_requests_in_progress)", + "legendFormat": "in-flight", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "Application Uptime", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "max(devops_info_uptime_seconds)", + "legendFormat": "uptime", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + } + }, + { + "id": 6, + "type": "heatmap", + "title": "Request Duration Heatmap", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (le) (rate(http_request_duration_seconds_bucket{endpoint!=\"/metrics\"}[5m]))", + "legendFormat": "{{le}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "calculate": false, + "legend": { + "show": false + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "unit": "s" + } + } + }, + { + "id": 7, + "type": "piechart", + "title": "Status Code Distribution", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (status_code) (rate(http_requests_total[5m]))", + "legendFormat": "{{status_code}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "pieType": "pie", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + } + } + }, + { + "id": 8, + "type": "stat", + "title": "App Target Uptime", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "expr": "up{job=\"app\"}", + "legendFormat": "app", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "textMode": "auto" + } + }, + { + "id": 9, + "type": "timeseries", + "title": "System Info Collection p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(devops_info_system_collection_seconds_bucket[5m])))", + "legendFormat": "system info p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + } + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} diff --git a/monitoring/grafana/provisioning/datasources/loki.yml b/monitoring/grafana/provisioning/datasources/loki.yml index 9c54f7c886..abeeb463cf 100644 --- a/monitoring/grafana/provisioning/datasources/loki.yml +++ b/monitoring/grafana/provisioning/datasources/loki.yml @@ -3,6 +3,8 @@ apiVersion: 1 deleteDatasources: - name: Loki orgId: 1 + - name: Prometheus + orgId: 1 prune: true @@ -17,3 +19,15 @@ datasources: jsonData: maxLines: 1000 timeout: 60 + + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: false + editable: false + jsonData: + httpMethod: POST + prometheusType: Prometheus + timeInterval: 15s diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000000..405abc0fdf --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,27 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + + - job_name: app + metrics_path: /metrics + static_configs: + - targets: + - app-python:8000 + + - job_name: loki + metrics_path: /metrics + static_configs: + - targets: + - loki:3100 + + - job_name: grafana + metrics_path: /metrics + static_configs: + - targets: + - grafana:3000