Monitoring?

5 files changed, 351 insertions, 0 deletions

diff --git a/modules/monitoring/crutches/synapse.nix b/modules/monitoring/crutches/synapse.nix
new file mode 100644
index 0000000..3145d11
--- /dev/null
+++ b/modules/monitoring/crutches/synapse.nix
@@ -0,0 +1,11 @@
+{lib, ...}:
+
+{
+  options.monitoring.synapse = {
+    workerNames = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [];
+      description = "Synapse worker names";
+    };
+  };
+}
diff --git a/modules/monitoring/module.nix b/modules/monitoring/module.nix
new file mode 100644
index 0000000..60e396d
--- /dev/null
+++ b/modules/monitoring/module.nix
@@ -0,0 +1,106 @@
+{ lib, config, ... }:
+let
+  mkStringOption =
+    description:
+    lib.mkOption {
+      type = lib.types.str;
+      default = null;
+      description = description;
+    };
+  cfg = config.monitoring;
+in
+{
+  imports = [
+    # crutches due to nix limitations:
+    ./crutches/synapse.nix
+    
+    # modules
+    ./system.nix
+    ./postgres.nix
+    ./synapse.nix
+  ];
+  options.monitoring = {
+    monitorAll = lib.mkEnableOption "Monitor all services";
+    localPrometheus = lib.mkEnableOption "Local Prometheus";
+    exposePrometheus = lib.mkEnableOption "Expose Prometheus";
+    localGrafana = lib.mkEnableOption "Local Grafana";
+    exposeGrafana = lib.mkEnableOption "Expose Grafana";
+
+    nginxHost = mkStringOption "The virtual host name";
+    nginxSsl = lib.mkEnableOption "Enable SSL for Nginx";
+    prometheusScrapeIntervals = lib.mkOption {
+      type = lib.types.listOf lib.types.int;
+      default = [
+        1
+        2
+        5
+        15
+      ];
+      description = "Scrape intervals for Prometheus";
+    };
+  };
+
+  config = lib.mkIf (cfg.monitorAll) {
+    services = {
+      prometheus = lib.mkIf (cfg.localPrometheus) {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+      grafana = lib.mkIf (cfg.localGrafana) {
+        enable = true;
+        settings = {
+          server = {
+            domain = cfg.nginxHost;
+            enable_gzip = true;
+            protocol = "socket";
+            socket_mode = "0666";
+          };
+        };
+        provision = {
+          datasources.settings = {
+            apiVersion = 1;
+            #            datasources = [
+            #              {
+            #                name = "Prometheus";
+            #                type = "prometheus";
+            #                access = "proxy";
+            #                url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+            #                isDefault = true;
+            #              }
+            #            ];
+
+            datasources = lib.map (interval: {
+              name = "Prometheus-${toString interval}";
+              type = "prometheus";
+              access = "proxy";
+              url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+              #isDefault = true;
+              jsonData.timeInterval = "${toString interval}s";
+            }) cfg.prometheusScrapeIntervals;
+          };
+        };
+      };
+      nginx.virtualHosts = {
+        "${cfg.nginxHost}" = {
+          enableACME = cfg.nginxSsl;
+          addSSL = cfg.nginxSsl;
+          http3 = cfg.nginxSsl;
+          http3_hq = cfg.nginxSsl;
+          kTLS = cfg.nginxSsl;
+          locations = {
+            "/" = if cfg.exposeGrafana then { proxyPass = "http://unix:${config.services.grafana.settings.server.socket}"; } else { return = "200 'OK'"; };
+          };
+        };
+
+        "prometheus.${cfg.nginxHost}" = lib.mkIf (cfg.exposePrometheus) {
+          enableACME = cfg.nginxSsl;
+          addSSL = cfg.nginxSsl;
+          http3 = cfg.nginxSsl;
+          http3_hq = cfg.nginxSsl;
+          kTLS = cfg.nginxSsl;
+          locations."/".proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}";
+        };
+      };
+    };
+  };
+}
diff --git a/modules/monitoring/postgres.nix b/modules/monitoring/postgres.nix
new file mode 100644
index 0000000..bbb55e6
--- /dev/null
+++ b/modules/monitoring/postgres.nix
@@ -0,0 +1,48 @@
+{ lib, config, ... }:
+let
+  cfg = config.monitoring;
+in
+{
+  config = lib.mkIf (cfg.monitorAll && config.services.postgresql.enable) {
+    services.prometheus.exporters.postgres = {
+      enable = true;
+      extraFlags = [
+        "--collector.database_wraparound"
+        "--collector.long_running_transactions"
+        "--collector.postmaster"
+        "--collector.process_idle"
+        "--collector.stat_activity_autovacuum"
+        "--collector.stat_statements"
+        #"--collector.stat_wal_receiver" #we dont have WAL receivers
+        "--collector.statio_user_indexes"
+        "--collector.xlog_location"
+      ];
+    };
+
+    services.prometheus.scrapeConfigs = (
+      lib.map (interval: {
+        job_name = "postgres-${toString interval}s";
+        scrape_interval = "${toString interval}s";
+        static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ]; } ];
+      }) cfg.prometheusScrapeIntervals
+    );
+
+    services.grafana.provision.dashboards.settings = {
+      apiVersion = 1;
+      providers = [
+        {
+          name = "14114-postgres-overview";
+          orgId = 1;
+          type = "file";
+          options = {
+            path = builtins.fetchurl {
+              url = "https://grafana.com/api/dashboards/14114/revisions/1/download";
+              sha256 = "0qza4j8lywrj08bqbww52dgh2p2b9rkhq5p313g72i57lrlkacfl";
+            };
+          };
+        }
+      ];
+    };
+
+  };
+}
diff --git a/modules/monitoring/synapse.nix b/modules/monitoring/synapse.nix
new file mode 100644
index 0000000..12c5562
--- /dev/null
+++ b/modules/monitoring/synapse.nix
@@ -0,0 +1,111 @@
+{ lib, config, ... }:
+let
+  cfg = config.monitoring;
+in
+{
+  config = lib.mkIf (cfg.monitorAll && config.services.matrix-synapse.enable) {
+    services.matrix-synapse.settings.listeners = [
+      {
+        type = "http";
+        port = 9200;
+        resources = [
+          {
+            names = [ "metrics" ];
+            compress = false;
+          }
+        ];
+      }
+    ];
+
+    #services.matrix-synapse.workers = (
+    #  let
+    #    workerNames = config.services.matrix-synapse.workers;
+
+    #    hasMetricsListener =
+    #      (workerName:
+    #      lib.any (
+    #        listener: listener.type == "http" && (lib.any (resourceName: resourceName == "metrics") listener.resources)
+    #      ) config.services.matrix-synapse.workers.${workerName}.worker_listeners);
+
+    #    workerNamesWithoutMetrics = lib.traceVal lib.filter (workerName: (!hasMetricsListener workerName)) workerNames;
+    #  in
+    #  lib.listToAttrs (
+    #    lib.imap (index: workerName: {
+    #      name = workerName;
+    #      value = {
+    #        worker_listeners = [
+    #          {
+    #            type = "http";
+    #            port = 9200 + index;
+    #            resources = [
+    #              {
+    #                names = [ "metrics" ];
+    #                compress = false;
+    #              }
+    #            ];
+    #          }
+    #        ];
+    #      };
+    #    }) workerNamesWithoutMetrics
+    #  )
+    #);
+
+    services.matrix-synapse.workers = (
+      lib.listToAttrs (
+        lib.imap (index: workerName: {
+          name = workerName;
+          value = {
+            worker_listeners = [
+              {
+                type = "http";
+                port = 9200 + index + 1;
+                resources = [
+                  {
+                    names = [ "metrics" ];
+                    compress = false;
+                  }
+                ];
+              }
+            ];
+          };
+        }) config.monitoring.synapse.workerNames
+      )
+    );
+
+    services.prometheus.scrapeConfigs = (
+      (lib.map (interval: {
+        job_name = "synapse-main-${toString interval}s";
+        scrape_interval = "${toString interval}s";
+        static_configs = [ { targets = [ "localhost:9200" ]; } ];
+      }) cfg.prometheusScrapeIntervals)
+      ++ lib.flatten (
+        lib.imap (
+          index: workerName:
+          lib.map (interval: {
+            job_name = "synapse-${workerName}-${toString interval}s";
+            scrape_interval = "${toString interval}s";
+            static_configs = [ { targets = [ "localhost:${toString (9200 + index + 1)}" ]; } ];
+          }) cfg.prometheusScrapeIntervals
+        ) config.monitoring.synapse.workerNames
+      )
+    );
+
+    services.grafana.provision.dashboards.settings = {
+      apiVersion = 1;
+      providers = [
+        {
+          name = "matrix-synapse";
+          orgId = 1;
+          type = "file";
+          options = {
+            path = builtins.fetchurl {
+              url = "https://raw.githubusercontent.com/element-hq/synapse/master/contrib/grafana/synapse.json";
+              sha256 = "07qlr0waw9phmyd38bv22bn5v303w3397b89l44l3lzwhpnhs16s";
+            };
+          };
+        }
+      ];
+    };
+
+  };
+}
diff --git a/modules/monitoring/system.nix b/modules/monitoring/system.nix
new file mode 100644
index 0000000..cc1ec66
--- /dev/null
+++ b/modules/monitoring/system.nix
@@ -0,0 +1,75 @@
+{ lib, config, ... }:
+let
+  cfg = config.monitoring;
+in
+{
+  config = lib.mkIf (cfg.monitorAll) {
+    services.prometheus.exporters.node = {
+      enable = true;
+      port = 9100;
+      enabledCollectors = [
+        #"logind" #too slow
+        "systemd"
+        "processes"
+        "interrupts"
+        # Testing:
+        "buddyinfo"
+        "cgroups"
+        "ksmd"
+        "lnstat"
+        "mountstats"
+        "network_route"
+        #"perf" # requires sysctl change
+        "qdisc"
+        "sysctl"
+        "softirqs"
+        "tcpstat"
+      ];
+      disabledCollectors = [
+        "textfile"
+        "xfs"
+        "zfs"
+        "selinux"
+        "cpufreq"
+        "btrfs"
+        "powersupplyclass"
+        "mdadm"
+        "tapestats"
+        "fibrechannel"
+        "cpu_vulnerabilities"
+        "watchdog"
+        "thermal_zone"
+        "logind"
+        "nfs"
+        "nfsd"
+        "infiniband"
+      ];
+    };
+
+    services.prometheus.scrapeConfigs = (
+        lib.map (interval: {
+            job_name = "node-${toString interval}s";
+            scrape_interval = "${toString interval}s";
+            static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; } ];
+        }) cfg.prometheusScrapeIntervals
+    );
+
+    services.grafana.provision.dashboards.settings = {
+      apiVersion = 1;
+      providers = [
+        {
+          name = "default";
+          orgId = 1;
+          type = "file";
+          options = {
+            path = builtins.fetchurl {
+              url = "https://grafana.com/api/dashboards/1860/revisions/37/download";
+              sha256 = "0qza4j8lywrj08bqbww52dgh2p2b9rkhq5p313g72i57lrlkacfl";
+            };
+          };
+        }
+      ];
+    };
+
+  };
+}