Rename forrest role to prometheus

Makes organising much simpler
2024-04-21 19:47:02 +01:00
parent ffbba254fb
commit ee96e6ab08
12 changed files with 1 additions and 1 deletions
--- a/ansible/roles/prometheus/files/grafana/docker-compose.yml
+++ b/ansible/roles/prometheus/files/grafana/docker-compose.yml
@ -0,0 +1,57 @@
+version: "2.3"
+
+services:
+  grafana:
+    image: grafana/grafana:latest
+    environment:
+      - TZ={{ timezone }}
+      - GF_DATABASE_URL=postgres://grafana:grafana@db/grafana
+      - GF_RENDERING_SERVER_URL=http://renderer:8081/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000/
+      - GF_SERVER_ROOT_URL=https://grafana.jakehoward.tech
+      - GF_SERVER_ENABLE_GZIP=true
+      - GF_DEFAULT_FORCE_MIGRATION=true
+      - GF_FEATURE_TOGGLES_ENABLE=publicDashboards
+
+      - GF_ANALYTICS_ENABLED=false
+      - GF_ANALYTICS_REPORTING_ENABLED=false
+      - GF_ANALYTICS_CHECK_FOR_UPDATES=false
+      - GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES=false
+
+      - GF_SMTP_ENABLED=true
+      - GF_SMTP_HOST=smtp.eu.mailgun.org:465
+      - GF_SMTP_USER={{ vault_grafana_smtp_user }}
+      - GF_SMTP_PASSWORD={{ vault_grafana_smtp_password }}
+      - GF_SMTP_FROM_ADDRESS={{ vault_grafana_from_email }}
+      - GF_SMTP_FROM_NAME=grafana
+    volumes:
+      - "{{ app_data_dir }}/grafana:/var/lib/grafana"
+    networks:
+      - default
+      - grafana
+    restart: unless-stopped
+    ports:
+      - "{{ pve_hosts.forrest.ip }}:3000:3000"
+    depends_on:
+      - db
+      - renderer
+
+  db:
+    image: postgres:14-alpine
+    restart: unless-stopped
+    volumes:
+      - /mnt/speed/dbs/postgres/grafana/:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=grafana
+      - POSTGRES_USER=grafana
+
+  renderer:
+    image: grafana/grafana-image-renderer:latest
+    restart: unless-stopped
+    environment:
+      - BROWSER_TZ={{ timezone }}
+
+
+networks:
+  grafana:
+    external: true
--- a/ansible/roles/prometheus/files/prometheus/alert-rules.d/blackbox.yml
+++ b/ansible/roles/prometheus/files/prometheus/alert-rules.d/blackbox.yml
@ -0,0 +1,6 @@
+groups:
+  - name: blackbox
+    rules:
+      - alert: HttpSuccess
+        expr: probe_success{job="blackbox_http_external"} == 0
+        for: 5m
--- a/ansible/roles/prometheus/files/prometheus/alertmanager.yml
+++ b/ansible/roles/prometheus/files/prometheus/alertmanager.yml
@ -0,0 +1,15 @@
+global:
+  resolve_timeout: 3m
+  smtp_smarthost: smtp.eu.mailgun.org:465
+  smtp_from: "{{ vault_alertmanager_from_address }}"
+  smtp_auth_username: "{{ vault_alertmanager_from_address }}"
+  smtp_auth_password: "{{ vault_alertmanager_smtp_password }}"
+
+route:
+  receiver: default
+
+receivers:
+  - name: default
+    email_configs:
+      - to: "{{ vault_alertmanager_to_address }}"
+        send_resolved: true
--- a/ansible/roles/prometheus/files/prometheus/blackbox.yml
+++ b/ansible/roles/prometheus/files/prometheus/blackbox.yml
@ -0,0 +1,19 @@
+modules:
+  http:
+    prober: http
+    timeout: 10s
+
+  https_redir:
+    prober: http
+    timeout: 10s
+    http:
+      method: GET
+      valid_status_codes: [301, 302, 307, 308]
+      follow_redirects: false
+      fail_if_ssl: true
+      fail_if_header_not_matches:
+        - header: Location
+          regexp: ^https
+
+  icmp:
+    prober: icmp
--- a/ansible/roles/prometheus/files/prometheus/docker-compose.yml
+++ b/ansible/roles/prometheus/files/prometheus/docker-compose.yml
@ -0,0 +1,60 @@
+version: "2.3"
+
+x-blackbox: &blackbox
+  image: prom/blackbox-exporter:latest
+  restart: unless-stopped
+  user: "{{ docker_user.id }}"
+  volumes:
+    - ./blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    restart: unless-stopped
+    user: "{{ docker_user.id }}"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./alert-rules.d:/etc/prometheus/alert-rules.d:ro
+      - /mnt/speed/dbs/prometheus/forrest/:/prometheus/
+    networks:
+      - default
+      - grafana
+    ports:
+      - "{{ pve_hosts.forrest.ip }}:9090:9090"
+
+  blackbox:
+    <<: *blackbox
+
+  blackbox-external:
+    <<: *blackbox
+    # Don't use my internal DNS servers
+    dns:
+      - 9.9.9.9
+      - 149.112.112.112
+
+  alertmanager:
+    image: prom/alertmanager:latest
+    restart: unless-stopped
+    volumes:
+      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+
+  proxmox_exporter:
+    image: prompve/prometheus-pve-exporter:latest
+    restart: unless-stopped
+    user: "{{ docker_user.id }}"
+    environment:
+      - PVE_USER=prometheus@pve
+      - PVE_TOKEN_NAME=prometheus
+      - PVE_TOKEN_VALUE={{ vault_prometheus_api_token }}
+      - PVE_VERIFY_SSL=false
+
+  speedtest_exporter:
+    image: jraviles/prometheus_speedtest:latest
+    restart: unless-stopped
+    user: "{{ docker_user.id }}"
+
+networks:
+  grafana:
+    external: true
+  default:
+    enable_ipv6: true
--- a/ansible/roles/prometheus/files/prometheus/prometheus.yml
+++ b/ansible/roles/prometheus/files/prometheus/prometheus.yml
@ -0,0 +1,169 @@
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+rule_files:
+  - /etc/prometheus/alert-rules.d/*.yml
+
+scrape_configs:
+  - job_name: pve
+    static_configs:
+      - targets:
+          - "{{ pve_hosts.pve.ip }}"
+    metrics_path: /pve
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: proxmox_exporter:9221
+
+  - job_name: traefik
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: go_.+
+        action: drop
+    static_configs:
+      - targets:
+          - "{{ pve_hosts.docker.ip }}:8080"
+
+  - job_name: homeassistant
+    metrics_path: /api/prometheus
+    authorization:
+      credentials: "{{ vault_homeassistant_token }}"
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: python_.+
+        action: drop
+    static_configs:
+      - targets: ["{{ pve_hosts.homeassistant.ip }}:8123"]
+
+  - job_name: blackbox_http_external
+    scrape_interval: 1m
+    metrics_path: /probe
+    params:
+      module: [http]
+    static_configs:
+      - targets:
+          - https://0rng.one
+          - https://auth.jakehoward.tech/-/health/ready/
+          - https://bin.theorangeone.net
+          - https://git.theorangeone.net/api/healthz
+          - https://grafana.jakehoward.tech/api/health
+          - https://headscale.jakehoward.tech/health
+          - https://homeassistant.jakehoward.tech
+          - https://intersect.jakehoward.tech
+          - https://mastodon.theorangeone.net/health
+          - https://matrix.jakehoward.tech:8448/_matrix/federation/v1/version
+          - https://matrix.jakehoward.tech/_matrix/federation/v1/version
+          - https://media.jakehoward.tech
+          - https://minio.jakehoward.tech/minio/health/live
+          - https://notes.theorangeone.net
+          - https://ntfy.jakehoward.tech/v1/health
+          - https://plausible.theorangeone.net
+          - https://recipes.jakehoward.tech
+          - https://s3.jakehoward.tech/minio/health/live
+          - https://tasks.jakehoward.tech/health
+          - https://theorangeone.net
+          - https://tt-rss.jakehoward.tech
+          - https://vaultwarden.jakehoward.tech/alive
+          - https://whoami-cdn.theorangeone.net
+          - https://whoami.theorangeone.net
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-external:9115
+      - source_labels: [instance]
+        regex: https?://([^/]+)/?.*
+        target_label: hostname
+
+  - job_name: blackbox_icmp
+    scrape_interval: 10m
+    metrics_path: /probe
+    params:
+      module: [icmp]
+    static_configs:
+      - targets:
+          - "{{ pve_hosts.pve.ip }}"
+          - "{{ pve_hosts.homeassistant.ip }}"
+          - 9.9.9.9
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: blackbox_https_redir
+    scrape_interval: 10m
+    metrics_path: /probe
+    params:
+      module: [https_redir]
+    static_configs:
+      - targets:
+          - http://{{ vps_hosts.casey_ip }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-external:9115
+
+  - job_name: blackbox_healthcheck
+    scrape_interval: 10m
+    metrics_path: /probe
+    params:
+      module: [http]
+    static_configs:
+      - targets:
+          - https://hc-ping.com/{{ vault_prometheus_healthcheck_uuid }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: healthchecks
+    scheme: https
+    metrics_path: /projects/{{ vault_healthchecks_project_uuid }}/metrics/{{ vault_healthcheck_api_token }}
+    static_configs:
+      - targets: [healthchecks.io]
+
+  - job_name: node
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - "{{ pve_hosts.pve.ip }}:9100"
+          - 192.168.1.53:9100  # adguardhome
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: go_.+
+        action: drop
+
+  - job_name: speedtest
+    scrape_interval: 1h
+    scrape_timeout: 2m
+    metrics_path: /probe
+    static_configs:
+      - targets:
+          - speedtest_exporter:9516
+
+  - job_name: headscale
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - "{{ nebula.clients.casey.ip }}:9090"
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: go_.+
+        action: drop
--- a/ansible/roles/prometheus/handlers/main.yml
+++ b/ansible/roles/prometheus/handlers/main.yml
@ -0,0 +1,14 @@
+- name: restart grafana
+  shell:
+    chdir: /opt/grafana
+    cmd: "{{ docker_update_command }}"
+
+- name: restart prometheus
+  shell:
+    chdir: /opt/prometheus
+    cmd: "{{ docker_update_command }}"
+
+- name: reload prometheus
+  shell:
+    chdir: /opt/prometheus
+    cmd: docker-compose exec prometheus kill -HUP 1
--- a/ansible/roles/prometheus/tasks/grafana.yml
+++ b/ansible/roles/prometheus/tasks/grafana.yml
@ -0,0 +1,21 @@
+- name: Create network
+  docker_network:
+    name: grafana
+
+- name: Create grafana install directory
+  file:
+    path: /opt/grafana
+    state: directory
+    owner: "{{ docker_user.name }}"
+    mode: "{{ docker_compose_directory_mask }}"
+  become: true
+
+- name: Install grafana compose file
+  template:
+    src: files/grafana/docker-compose.yml
+    dest: /opt/grafana/docker-compose.yml
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+    validate: docker-compose -f %s config
+  notify: restart grafana
+  become: true
--- a/ansible/roles/prometheus/tasks/main.yml
+++ b/ansible/roles/prometheus/tasks/main.yml
@ -0,0 +1,35 @@
+- name: Include vault
+  include_vars: vault.yml
+
+- name: Grafana
+  include_tasks: grafana.yml
+
+- name: Prometheus
+  include_tasks: prometheus.yml
+
+- name: Get routes
+  command:
+    argv:
+      - ip
+      - -6
+      - route
+      - show
+      - "{{ vps_hosts.private_ipv6_range }}"
+  register: routes
+  changed_when: false
+  become: true
+
+- name: Add route to private services via ingress
+  command:
+    argv:
+      - ip
+      - -6
+      - route
+      - add
+      - "{{ vps_hosts.private_ipv6_range }}"
+      - via
+      - "{{ pve_hosts.ingress.ipv6 }}"
+      - dev
+      - eth0
+  become: true
+  when: vps_hosts.private_ipv6_marker not in routes.stdout
--- a/ansible/roles/prometheus/tasks/prometheus.yml
+++ b/ansible/roles/prometheus/tasks/prometheus.yml
@ -0,0 +1,53 @@
+- name: Create prometheus install directory
+  file:
+    path: /opt/prometheus
+    state: directory
+    owner: "{{ docker_user.name }}"
+    mode: "{{ docker_compose_directory_mask }}"
+  become: true
+
+- name: Install prometheus config
+  template:
+    src: files/prometheus/prometheus.yml
+    dest: /opt/prometheus/prometheus.yml
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+  notify: reload prometheus
+  become: true
+
+- name: Install prometheus compose file
+  template:
+    src: files/prometheus/docker-compose.yml
+    dest: /opt/prometheus/docker-compose.yml
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+    validate: docker-compose -f %s config
+  notify: restart prometheus
+  become: true
+
+- name: Install blackbox config
+  template:
+    src: files/prometheus/blackbox.yml
+    dest: /opt/prometheus/blackbox.yml
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+  notify: restart prometheus
+  become: true
+
+- name: Install alertmanager config
+  template:
+    src: files/prometheus/alertmanager.yml
+    dest: /opt/prometheus/alertmanager.yml
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+  notify: restart prometheus
+  become: true
+
+- name: Install prometheus alert rules
+  copy:
+    src: files/prometheus/alert-rules.d/
+    dest: /opt/prometheus/alert-rules.d/
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+  notify: reload prometheus
+  become: true
--- a/ansible/roles/prometheus/vars/vault.yml
+++ b/ansible/roles/prometheus/vars/vault.yml
@ -0,0 +1,52 @@
+$ANSIBLE_VAULT;1.1;AES256
+36623535313964653161353330663436356239613837653837393939373034353031646535333535
+6439313832316239616233306632373934616134616466380a316361363263373938636161666535
+31613461333637373732626233623434316335353964353433643635653566613933393361336139
+3864373963396264320a376634346331373762313733323961386566646338633936303631303566
+66616534326430653266396635353932623661363533356537636662636537656434363562646230
+30613831336561376639393466373739373138313931333163353061633465623362666564313631
+66623235353531613737643937613430323934376433393836346339626137616561313062663234
+63363736326439623661376132613136383465393761653236663631613339653066356436653630
+66623865303735616335373231643233386639323838353534613337316161633765396234366533
+33616631663530643764373937346262633734366339303837393737666665363465333239343933
+35613962396534336232623833303034643639323931633966396439383463396261313862626335
+31323434613838353961336136613966636635646632393839663664376632373834313265643338
+30663132633362323831313231333164643665386535323231646262656631383631393539616639
+34343563353064303833383236626136666264316236316537333965313162616637323966363335
+32353936663162316564306337353861396634353935353935306135343665316262643831396537
+61393266383538666563363261646534636632303332343662636631316663343930303766623638
+35376565343638316339623061396536643636313966383633346231633631353032356661386132
+66623439336338616666626431303635373833666137326234653161336434346133636261363662
+39313732303736386137656664303365363234336265643064306562643435633838373864353862
+33366635333630373162656630666232333563623066333461653963363961623435646631373561
+64643738346138366566303233326663383835386132663034313461383161616164636332396332
+37663131386135393833373461663432666264363065666630646164633134303439663435616235
+35656234313761376532306264393637653433623863383830323935316332383338623134323366
+31336665386137323132363962363335623635336131373930353635353663333366363266303138
+35626262613261636561373730626635303836623561643436646430653365663432323938393863
+63633331663462323163646237386262376337313330323036613434383165616530643362616131
+63616562353964316634646434653138333266646633616631653663663838306163616633643234
+61333230373237613436343662363434303766383336376232353066313231666330613761643366
+36326638326439653966643430313366376661633636366565393461623438323366373333663633
+61633763623631333665363333646433656166633364303836623566333336343761613435353138
+37366165613263653564386334303030623333646164303662363065333831376334656537613130
+33373864663237383064653461616165653834393063663332643235316139333539623463343161
+38636564626466633631393938653066373764663935353763626133623762306164383831663061
+34333065326666373337663931313763383739383763333235333939376133363236643136346233
+62643833376631643036613963643939333133343036613332313866373032646332363231313139
+61373365653665343066636162356336373833393363373866343436323639623435383831363335
+30333033326638363930613030356664333233633339666366643062353634333161343838666231
+32346332663538653937623136653438636463323463376263303962353562313833373937303066
+65303037323030653434313164393766633134306435633263363335636561356264376665363639
+35613731373437386566663266656266343639326334303239613862353963323436633836383766
+35323930633039396535616265643234303639393035363865643236623838333337626135343665
+36373038666332376663333565623362303631663830336131343438353764653831633433363436
+36333839303433623966363561313564303037393165383732323763353232653564346138666438
+30653836626139356133346538616135313034633966373036303461393562363336386633626365
+33393565643730383634346238356462313435366538636234656237613864656165656439363061
+32626235323362333239373631383830653035383164646364343461376562636564343063353139
+61306535333466653937303635353962376162376431336563316130343530636431623537633332
+65373333376338353930316561636530343062653964323463653632653332376432343237656465
+63333437613064313438353134333566303033313339323162643061363836643931343135396130
+32623435653533326563616263323938343332306362383034663139653965626231336637383939
+313534343431303739396263303737303365