nix-gscheits/machines/raven/services/prometheus.nix

{ config, lib, ... }:

let
  domain = "prometheus.fablab-nea.de";
  cfg = config.services.prometheus;
  mkStaticTargets = targets: lib.singleton { inherit targets; };
  mkStaticTarget = target: mkStaticTargets (lib.singleton target);
in
{
  sops.secrets.prometheus-htpasswd = {
    owner = "nginx";
    sopsFile = ../secrets.yaml;
  };

  services = {

    prometheus = {
      exporters.node.enable = true;
      enable = true;
      listenAddress = "127.0.0.1";
      webExternalUrl = "https://${domain}";
      globalConfig = {
        scrape_interval = "15s";
        evaluation_interval = "15s";
      };
      extraFlags = [
        "--storage.tsdb.retention.time=90d"
        "--web.enable-admin-api"
      ];
      alertmanagers = [
        {
          static_configs = mkStaticTarget "${cfg.alertmanager.listenAddress}:${toString cfg.alertmanager.port}";
          path_prefix = "/alertmanager/";
        }
      ];
      alertmanager = {
        enable = true;
        listenAddress = "127.0.0.1";
        webExternalUrl = "https://${domain}/alertmanager";
        configuration = {
          global.resolve_timeout = "2m";

          route = {
            receiver = "matrix";
            group_by = [ "alertname" ];
            group_wait = "3m";
          };

          receivers = [
            {
              name = "matrix";
              webhook_configs = lib.singleton {
                url = "http://localhost/webhook";
              };
            }
          ];
        };
      };
      scrapeConfigs = [
        {
          job_name = "prometheus";
          static_configs = mkStaticTargets [
            "localhost:${toString cfg.port}"
            "kleinturmbuehne-router:9100"
          ];
        }
        {
          job_name = "node";
          static_configs = mkStaticTargets [
            "127.0.0.1:9100"
          ];
        }
        {
          job_name = "asterisk";
          metrics_path = "/";
          static_configs = mkStaticTargets [
            "127.0.0.1:8088"
          ];
        }
        {
          job_name = "mikrotik";
          static_configs = mkStaticTargets [
            "${cfg.exporters.mikrotik.listenAddress}:${toString cfg.exporters.mikrotik.port}"
          ];
        }
        {
          job_name = "unifi";
          static_configs = mkStaticTargets [
            "${cfg.exporters.unpoller.listenAddress}:${toString cfg.exporters.unpoller.port}"
          ];
        }
      ];
      rules =
        let
          mkAlert = { name, expr, for ? "1m", description ? null }: {
            alert = name;
            inherit expr for;
            annotations = lib.optionalAttrs (description != null) { inherit description; };
          };
        in
        [
          (lib.generators.toYAML { } {
            groups = lib.singleton {
              name = "alert.rules";
              rules = map mkAlert [
                {
                  name = "InstanceDown";
                  expr = ''up == 0'';
                  description = "Instance {{ $labels.instance }} of job {{ $labels.job }} has been down for
   more than 1 minutes.";
                }
              ];
            };
          })
        ];
    };

    prometheus.exporters.mikrotik = {
      enable = true;
      listenAddress = "127.0.0.1";
      configuration = {
        devices = [
        ];
        features = {
          bgp = true;
          dhcp = true;
          routes = true;
          optics = true;
        };
      };
    };

    nginx.virtualHosts."${domain}" = {
      enableACME = true;
      forceSSL = true;

      basicAuthFile = config.sops.secrets.prometheus-htpasswd.path;

      locations = {
        "/".proxyPass = "http://${cfg.listenAddress}:${toString cfg.port}";

        "/alertmanager/".proxyPass = "http://${cfg.alertmanager.listenAddress}:${toString cfg.alertmanager.port}";
      };
    };
  };
}