Rename forrest role to prometheus

Makes organising much simpler

ansible/roles/prometheus/files/prometheus/alert-rules.d/blackbox.yml

@@ -0,0 +1,6 @@
+groups:
+  - name: blackbox
+    rules:
+      - alert: HttpSuccess
+        expr: probe_success{job="blackbox_http_external"} == 0
+        for: 5m

ansible/roles/prometheus/files/prometheus/alertmanager.yml

@@ -0,0 +1,15 @@
+global:
+  resolve_timeout: 3m
+  smtp_smarthost: smtp.eu.mailgun.org:465
+  smtp_from: "{{ vault_alertmanager_from_address }}"
+  smtp_auth_username: "{{ vault_alertmanager_from_address }}"
+  smtp_auth_password: "{{ vault_alertmanager_smtp_password }}"
+
+route:
+  receiver: default
+
+receivers:
+  - name: default
+    email_configs:
+      - to: "{{ vault_alertmanager_to_address }}"
+        send_resolved: true

ansible/roles/prometheus/files/prometheus/blackbox.yml

@@ -0,0 +1,19 @@
+modules:
+  http:
+    prober: http
+    timeout: 10s
+
+  https_redir:
+    prober: http
+    timeout: 10s
+    http:
+      method: GET
+      valid_status_codes: [301, 302, 307, 308]
+      follow_redirects: false
+      fail_if_ssl: true
+      fail_if_header_not_matches:
+        - header: Location
+          regexp: ^https
+
+  icmp:
+    prober: icmp

ansible/roles/prometheus/files/prometheus/docker-compose.yml

@@ -0,0 +1,60 @@
+version: "2.3"
+
+x-blackbox: &blackbox
+  image: prom/blackbox-exporter:latest
+  restart: unless-stopped
+  user: "{{ docker_user.id }}"
+  volumes:
+    - ./blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    restart: unless-stopped
+    user: "{{ docker_user.id }}"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./alert-rules.d:/etc/prometheus/alert-rules.d:ro
+      - /mnt/speed/dbs/prometheus/forrest/:/prometheus/
+    networks:
+      - default
+      - grafana
+    ports:
+      - "{{ pve_hosts.forrest.ip }}:9090:9090"
+
+  blackbox:
+    <<: *blackbox
+
+  blackbox-external:
+    <<: *blackbox
+    # Don't use my internal DNS servers
+    dns:
+      - 9.9.9.9
+      - 149.112.112.112
+
+  alertmanager:
+    image: prom/alertmanager:latest
+    restart: unless-stopped
+    volumes:
+      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+
+  proxmox_exporter:
+    image: prompve/prometheus-pve-exporter:latest
+    restart: unless-stopped
+    user: "{{ docker_user.id }}"
+    environment:
+      - PVE_USER=prometheus@pve
+      - PVE_TOKEN_NAME=prometheus
+      - PVE_TOKEN_VALUE={{ vault_prometheus_api_token }}
+      - PVE_VERIFY_SSL=false
+
+  speedtest_exporter:
+    image: jraviles/prometheus_speedtest:latest
+    restart: unless-stopped
+    user: "{{ docker_user.id }}"
+
+networks:
+  grafana:
+    external: true
+  default:
+    enable_ipv6: true

ansible/roles/prometheus/files/prometheus/prometheus.yml

@@ -0,0 +1,169 @@
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+rule_files:
+  - /etc/prometheus/alert-rules.d/*.yml
+
+scrape_configs:
+  - job_name: pve
+    static_configs:
+      - targets:
+          - "{{ pve_hosts.pve.ip }}"
+    metrics_path: /pve
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: proxmox_exporter:9221
+
+  - job_name: traefik
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: go_.+
+        action: drop
+    static_configs:
+      - targets:
+          - "{{ pve_hosts.docker.ip }}:8080"
+
+  - job_name: homeassistant
+    metrics_path: /api/prometheus
+    authorization:
+      credentials: "{{ vault_homeassistant_token }}"
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: python_.+
+        action: drop
+    static_configs:
+      - targets: ["{{ pve_hosts.homeassistant.ip }}:8123"]
+
+  - job_name: blackbox_http_external
+    scrape_interval: 1m
+    metrics_path: /probe
+    params:
+      module: [http]
+    static_configs:
+      - targets:
+          - https://0rng.one
+          - https://auth.jakehoward.tech/-/health/ready/
+          - https://bin.theorangeone.net
+          - https://git.theorangeone.net/api/healthz
+          - https://grafana.jakehoward.tech/api/health
+          - https://headscale.jakehoward.tech/health
+          - https://homeassistant.jakehoward.tech
+          - https://intersect.jakehoward.tech
+          - https://mastodon.theorangeone.net/health
+          - https://matrix.jakehoward.tech:8448/_matrix/federation/v1/version
+          - https://matrix.jakehoward.tech/_matrix/federation/v1/version
+          - https://media.jakehoward.tech
+          - https://minio.jakehoward.tech/minio/health/live
+          - https://notes.theorangeone.net
+          - https://ntfy.jakehoward.tech/v1/health
+          - https://plausible.theorangeone.net
+          - https://recipes.jakehoward.tech
+          - https://s3.jakehoward.tech/minio/health/live
+          - https://tasks.jakehoward.tech/health
+          - https://theorangeone.net
+          - https://tt-rss.jakehoward.tech
+          - https://vaultwarden.jakehoward.tech/alive
+          - https://whoami-cdn.theorangeone.net
+          - https://whoami.theorangeone.net
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-external:9115
+      - source_labels: [instance]
+        regex: https?://([^/]+)/?.*
+        target_label: hostname
+
+  - job_name: blackbox_icmp
+    scrape_interval: 10m
+    metrics_path: /probe
+    params:
+      module: [icmp]
+    static_configs:
+      - targets:
+          - "{{ pve_hosts.pve.ip }}"
+          - "{{ pve_hosts.homeassistant.ip }}"
+          - 9.9.9.9
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: blackbox_https_redir
+    scrape_interval: 10m
+    metrics_path: /probe
+    params:
+      module: [https_redir]
+    static_configs:
+      - targets:
+          - http://{{ vps_hosts.casey_ip }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-external:9115
+
+  - job_name: blackbox_healthcheck
+    scrape_interval: 10m
+    metrics_path: /probe
+    params:
+      module: [http]
+    static_configs:
+      - targets:
+          - https://hc-ping.com/{{ vault_prometheus_healthcheck_uuid }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: healthchecks
+    scheme: https
+    metrics_path: /projects/{{ vault_healthchecks_project_uuid }}/metrics/{{ vault_healthcheck_api_token }}
+    static_configs:
+      - targets: [healthchecks.io]
+
+  - job_name: node
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - "{{ pve_hosts.pve.ip }}:9100"
+          - 192.168.1.53:9100  # adguardhome
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: go_.+
+        action: drop
+
+  - job_name: speedtest
+    scrape_interval: 1h
+    scrape_timeout: 2m
+    metrics_path: /probe
+    static_configs:
+      - targets:
+          - speedtest_exporter:9516
+
+  - job_name: headscale
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - "{{ nebula.clients.casey.ip }}:9090"
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: go_.+
+        action: drop