From 62fb7f7c127ca8aee56b3903ee207dec766d0a11 Mon Sep 17 00:00:00 2001 From: Rory& Date: Wed, 28 Aug 2024 21:40:53 +0200 Subject: Monitoring? --- flake.lock | 12 +-- flake.nix | 15 ++- host/Rory-desktop/configuration.nix | 18 +++- host/Rory-nginx/configuration.nix | 17 +++- .../services/matrix/synapse/workers/auth.nix | 6 +- .../matrix/synapse/workers/client-reader.nix | 6 +- .../matrix/synapse/workers/event-creator.nix | 6 +- .../matrix/synapse/workers/federation-inbound.nix | 6 +- .../matrix/synapse/workers/federation-reader.nix | 6 +- .../matrix/synapse/workers/federation-sender.nix | 6 +- .../services/matrix/synapse/workers/media-repo.nix | 6 +- .../services/matrix/synapse/workers/pusher.nix | 6 +- .../matrix/synapse/workers/single/appservice.nix | 6 +- .../matrix/synapse/workers/single/background.nix | 6 +- .../matrix/synapse/workers/single/user-dir.nix | 6 +- .../workers/stream-writers/event-stream-writer.nix | 6 +- .../services/matrix/synapse/workers/sync.nix | 6 +- modules/base.nix | 1 + modules/monitoring/crutches/synapse.nix | 11 ++ modules/monitoring/module.nix | 106 ++++++++++++++++++++ modules/monitoring/postgres.nix | 48 +++++++++ modules/monitoring/synapse.nix | 111 +++++++++++++++++++++ modules/monitoring/system.nix | 75 ++++++++++++++ modules/users/Rory.client.nix | 2 + 24 files changed, 466 insertions(+), 28 deletions(-) create mode 100644 modules/monitoring/crutches/synapse.nix create mode 100644 modules/monitoring/module.nix create mode 100644 modules/monitoring/postgres.nix create mode 100644 modules/monitoring/synapse.nix create mode 100644 modules/monitoring/system.nix diff --git a/flake.lock b/flake.lock index d00ae07..d86d743 100644 --- a/flake.lock +++ b/flake.lock @@ -404,11 +404,11 @@ }, "locked": { "host": "gitlab.computer.surgery", - "lastModified": 1724088992, - "narHash": "sha256-e45kLOGHA6PDDhYzq3dV4SsYf+hr2+n5SDnta9NrN14=", + "lastModified": 1724527899, + "narHash": "sha256-aLCrE4J9BTSoqWmiLQqQngt1ZNgsDZ2bOZgPRUbV2L4=", "owner": "matrix", "repo": "grapevine-fork", - "rev": "c355e2ad394a3c925acf24c87854e0898d0e328d", + "rev": "14afa1357e199e542835fde621e99052e7d12f2b", "type": "gitlab" }, "original": { @@ -572,11 +572,11 @@ }, "nixpkgs-master": { "locked": { - "lastModified": 1724475946, - "narHash": "sha256-rEseutXa9VP4HFiEChUX0cw7hbilCkMq/RBeKT3XPV4=", + "lastModified": 1724591773, + "narHash": "sha256-3G7MkSEWUIigBmpd5djDtfZfx7SfPj+crUw7gdrgO3k=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "77e2fa9398204bb8cda108ce1e56c282be7fdb9e", + "rev": "9e2335ac963ae6863194f516b57b01febd499e84", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index d34ccc4..bcef2ce 100755 --- a/flake.nix +++ b/flake.nix @@ -17,9 +17,7 @@ }; # Draupnir module/package - nixpkgs-Draupnir = { - url = "github:TheArcaneBrony/nixpkgs/master"; - }; + nixpkgs-Draupnir.url = "github:TheArcaneBrony/nixpkgs/master"; # Base modules home-manager.url = "github:nix-community/home-manager/master"; @@ -79,6 +77,17 @@ matrix-synapse-unwrapped = inputs.nixpkgs-master.legacyPackages.${pkgs.stdenv.hostPlatform.system}.matrix-synapse-unwrapped; draupnir = inputs.nixpkgs-Draupnir.legacyPackages.${pkgs.stdenv.hostPlatform.system}.draupnir; }) + + # // matrix-nio + (final: prev: { + python312 = prev.python312.override { + packageOverrides = python-self: python-super: { + matrix-nio = python-super.matrix-nio.overridePythonAttrs (old: { + pythonRelaxDeps = [ "aiohttp-socks" ]; + }); + }; + }; + }) ]; } ) diff --git a/host/Rory-desktop/configuration.nix b/host/Rory-desktop/configuration.nix index 3e21734..aa09659 100644 --- a/host/Rory-desktop/configuration.nix +++ b/host/Rory-desktop/configuration.nix @@ -24,8 +24,8 @@ args@{ ./optional/gui/x11.nix ./optional/gui/wayland.nix - ./printing.nix - #./ollama.nix + #./printing.nix +# ./ollama.nix ]; boot = { @@ -230,7 +230,7 @@ args@{ nixpkgs = { config = { allowUnfree = true; - permittedInsecurePackages = [ + permittedInsecurePackages = [ "electron-25.9.0" "olm-3.2.16" ]; @@ -262,5 +262,17 @@ args@{ virtualisation.libvirtd.enable = true; programs.virt-manager.enable = true; + monitoring = { + monitorAll = true; + localPrometheus = true; + exposePrometheus = true; + localGrafana = true; + exposeGrafana = true; + nginxHost = "monitoring.localhost"; + nginxSsl = false; + }; + networking.hosts."127.0.0.1" = builtins.attrNames config.services.nginx.virtualHosts; + + system.stateVersion = "22.11"; # DO NOT EDIT! } diff --git a/host/Rory-nginx/configuration.nix b/host/Rory-nginx/configuration.nix index ebcffd3..456e15c 100755 --- a/host/Rory-nginx/configuration.nix +++ b/host/Rory-nginx/configuration.nix @@ -1,5 +1,6 @@ { pkgs, + config, lib, grapevine, conduit, @@ -51,6 +52,18 @@ }; }; + monitoring = { + monitorAll = true; + localPrometheus = true; + exposePrometheus = true; + localGrafana = true; + exposeGrafana = true; + nginxHost = "monitoring.rory.gay"; + nginxSsl = true; + }; + + nixpkgs.config.permittedInsecurePackages = [ "olm-3.2.16" ]; + services.irqbalance.enable = true; environment.memoryAllocator.provider = "jemalloc"; @@ -60,9 +73,7 @@ # conduit = grapevine; #}; - containers."matrixunittests" = import ./services/containers/matrixunittests/container.nix { - inherit pkgs lib grapevine; - }; + containers."matrixunittests" = import ./services/containers/matrixunittests/container.nix { inherit pkgs lib grapevine; }; containers."matrixunittests-conduit" = import ./services/containers/matrixunittests-conduit/container.nix { inherit pkgs lib; diff --git a/host/Rory-nginx/services/matrix/synapse/workers/auth.nix b/host/Rory-nginx/services/matrix/synapse/workers/auth.nix index 50d52c6..3c8d1e9 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/auth.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/auth.nix @@ -23,10 +23,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.authWorkers > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix b/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix index 249084a..60c48a2 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix @@ -46,10 +46,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.clientReaders > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix b/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix index c841b93..115ce51 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix @@ -19,10 +19,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.eventCreators > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/federation-inbound.nix b/host/Rory-nginx/services/matrix/synapse/workers/federation-inbound.nix index 83ceec7..effaa69 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/federation-inbound.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/federation-inbound.nix @@ -12,10 +12,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.federationInboundWorkers > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/federation-reader.nix b/host/Rory-nginx/services/matrix/synapse/workers/federation-reader.nix index 6a16fb6..5b3d4bf 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/federation-reader.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/federation-reader.nix @@ -36,10 +36,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.federationReaders > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/federation-sender.nix b/host/Rory-nginx/services/matrix/synapse/workers/federation-sender.nix index cd482f9..07af127 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/federation-sender.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/federation-sender.nix @@ -11,10 +11,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.federationSenders > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix b/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix index 62f3e9c..a361390 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix @@ -21,10 +21,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.mediaRepoWorkers > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/pusher.nix b/host/Rory-nginx/services/matrix/synapse/workers/pusher.nix index 7388824..edf1632 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/pusher.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/pusher.nix @@ -11,10 +11,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.pushers > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/single/appservice.nix b/host/Rory-nginx/services/matrix/synapse/workers/single/appservice.nix index 315a322..119fd04 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/single/appservice.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/single/appservice.nix @@ -11,10 +11,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf cfg.enableAppserviceWorker { + monitoring.synapse.workerNames = [ workerName ]; services.matrix-synapse = { settings = { instance_map = { diff --git a/host/Rory-nginx/services/matrix/synapse/workers/single/background.nix b/host/Rory-nginx/services/matrix/synapse/workers/single/background.nix index b667e19..37fde10 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/single/background.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/single/background.nix @@ -13,10 +13,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf cfg.enableBackgroundWorker { + monitoring.synapse.workerNames = [ workerName ]; services.matrix-synapse = { settings = { instance_map = { diff --git a/host/Rory-nginx/services/matrix/synapse/workers/single/user-dir.nix b/host/Rory-nginx/services/matrix/synapse/workers/single/user-dir.nix index 89c6aa4..f26f3ec 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/single/user-dir.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/single/user-dir.nix @@ -16,10 +16,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf cfg.enableUserDirWorker { + monitoring.synapse.workerNames = [ workerName ]; services.matrix-synapse = { settings = { instance_map = { diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/event-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/event-stream-writer.nix index 35bd4e9..5395aea 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/event-stream-writer.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/event-stream-writer.nix @@ -13,10 +13,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.eventStreamWriters > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/host/Rory-nginx/services/matrix/synapse/workers/sync.nix b/host/Rory-nginx/services/matrix/synapse/workers/sync.nix index 636f521..6892847 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/sync.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/sync.nix @@ -17,10 +17,14 @@ let }; in let - enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; + enabledResources = + lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] + ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] + ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ]; in { config = lib.mkIf (cfg.syncWorkers > 0) { + monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers; services.matrix-synapse = { settings = { instance_map = lib.listToAttrs ( diff --git a/modules/base.nix b/modules/base.nix index bb122a8..4cf2aff 100755 --- a/modules/base.nix +++ b/modules/base.nix @@ -5,6 +5,7 @@ ./packages/vim.nix ./users/Rory.nix ./extra-substituters.nix + ./monitoring/module.nix ]; boot = { diff --git a/modules/monitoring/crutches/synapse.nix b/modules/monitoring/crutches/synapse.nix new file mode 100644 index 0000000..3145d11 --- /dev/null +++ b/modules/monitoring/crutches/synapse.nix @@ -0,0 +1,11 @@ +{lib, ...}: + +{ + options.monitoring.synapse = { + workerNames = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "Synapse worker names"; + }; + }; +} diff --git a/modules/monitoring/module.nix b/modules/monitoring/module.nix new file mode 100644 index 0000000..60e396d --- /dev/null +++ b/modules/monitoring/module.nix @@ -0,0 +1,106 @@ +{ lib, config, ... }: +let + mkStringOption = + description: + lib.mkOption { + type = lib.types.str; + default = null; + description = description; + }; + cfg = config.monitoring; +in +{ + imports = [ + # crutches due to nix limitations: + ./crutches/synapse.nix + + # modules + ./system.nix + ./postgres.nix + ./synapse.nix + ]; + options.monitoring = { + monitorAll = lib.mkEnableOption "Monitor all services"; + localPrometheus = lib.mkEnableOption "Local Prometheus"; + exposePrometheus = lib.mkEnableOption "Expose Prometheus"; + localGrafana = lib.mkEnableOption "Local Grafana"; + exposeGrafana = lib.mkEnableOption "Expose Grafana"; + + nginxHost = mkStringOption "The virtual host name"; + nginxSsl = lib.mkEnableOption "Enable SSL for Nginx"; + prometheusScrapeIntervals = lib.mkOption { + type = lib.types.listOf lib.types.int; + default = [ + 1 + 2 + 5 + 15 + ]; + description = "Scrape intervals for Prometheus"; + }; + }; + + config = lib.mkIf (cfg.monitorAll) { + services = { + prometheus = lib.mkIf (cfg.localPrometheus) { + enable = true; + listenAddress = "127.0.0.1"; + }; + grafana = lib.mkIf (cfg.localGrafana) { + enable = true; + settings = { + server = { + domain = cfg.nginxHost; + enable_gzip = true; + protocol = "socket"; + socket_mode = "0666"; + }; + }; + provision = { + datasources.settings = { + apiVersion = 1; + # datasources = [ + # { + # name = "Prometheus"; + # type = "prometheus"; + # access = "proxy"; + # url = "http://127.0.0.1:${toString config.services.prometheus.port}"; + # isDefault = true; + # } + # ]; + + datasources = lib.map (interval: { + name = "Prometheus-${toString interval}"; + type = "prometheus"; + access = "proxy"; + url = "http://127.0.0.1:${toString config.services.prometheus.port}"; + #isDefault = true; + jsonData.timeInterval = "${toString interval}s"; + }) cfg.prometheusScrapeIntervals; + }; + }; + }; + nginx.virtualHosts = { + "${cfg.nginxHost}" = { + enableACME = cfg.nginxSsl; + addSSL = cfg.nginxSsl; + http3 = cfg.nginxSsl; + http3_hq = cfg.nginxSsl; + kTLS = cfg.nginxSsl; + locations = { + "/" = if cfg.exposeGrafana then { proxyPass = "http://unix:${config.services.grafana.settings.server.socket}"; } else { return = "200 'OK'"; }; + }; + }; + + "prometheus.${cfg.nginxHost}" = lib.mkIf (cfg.exposePrometheus) { + enableACME = cfg.nginxSsl; + addSSL = cfg.nginxSsl; + http3 = cfg.nginxSsl; + http3_hq = cfg.nginxSsl; + kTLS = cfg.nginxSsl; + locations."/".proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}"; + }; + }; + }; + }; +} diff --git a/modules/monitoring/postgres.nix b/modules/monitoring/postgres.nix new file mode 100644 index 0000000..bbb55e6 --- /dev/null +++ b/modules/monitoring/postgres.nix @@ -0,0 +1,48 @@ +{ lib, config, ... }: +let + cfg = config.monitoring; +in +{ + config = lib.mkIf (cfg.monitorAll && config.services.postgresql.enable) { + services.prometheus.exporters.postgres = { + enable = true; + extraFlags = [ + "--collector.database_wraparound" + "--collector.long_running_transactions" + "--collector.postmaster" + "--collector.process_idle" + "--collector.stat_activity_autovacuum" + "--collector.stat_statements" + #"--collector.stat_wal_receiver" #we dont have WAL receivers + "--collector.statio_user_indexes" + "--collector.xlog_location" + ]; + }; + + services.prometheus.scrapeConfigs = ( + lib.map (interval: { + job_name = "postgres-${toString interval}s"; + scrape_interval = "${toString interval}s"; + static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ]; } ]; + }) cfg.prometheusScrapeIntervals + ); + + services.grafana.provision.dashboards.settings = { + apiVersion = 1; + providers = [ + { + name = "14114-postgres-overview"; + orgId = 1; + type = "file"; + options = { + path = builtins.fetchurl { + url = "https://grafana.com/api/dashboards/14114/revisions/1/download"; + sha256 = "0qza4j8lywrj08bqbww52dgh2p2b9rkhq5p313g72i57lrlkacfl"; + }; + }; + } + ]; + }; + + }; +} diff --git a/modules/monitoring/synapse.nix b/modules/monitoring/synapse.nix new file mode 100644 index 0000000..12c5562 --- /dev/null +++ b/modules/monitoring/synapse.nix @@ -0,0 +1,111 @@ +{ lib, config, ... }: +let + cfg = config.monitoring; +in +{ + config = lib.mkIf (cfg.monitorAll && config.services.matrix-synapse.enable) { + services.matrix-synapse.settings.listeners = [ + { + type = "http"; + port = 9200; + resources = [ + { + names = [ "metrics" ]; + compress = false; + } + ]; + } + ]; + + #services.matrix-synapse.workers = ( + # let + # workerNames = config.services.matrix-synapse.workers; + + # hasMetricsListener = + # (workerName: + # lib.any ( + # listener: listener.type == "http" && (lib.any (resourceName: resourceName == "metrics") listener.resources) + # ) config.services.matrix-synapse.workers.${workerName}.worker_listeners); + + # workerNamesWithoutMetrics = lib.traceVal lib.filter (workerName: (!hasMetricsListener workerName)) workerNames; + # in + # lib.listToAttrs ( + # lib.imap (index: workerName: { + # name = workerName; + # value = { + # worker_listeners = [ + # { + # type = "http"; + # port = 9200 + index; + # resources = [ + # { + # names = [ "metrics" ]; + # compress = false; + # } + # ]; + # } + # ]; + # }; + # }) workerNamesWithoutMetrics + # ) + #); + + services.matrix-synapse.workers = ( + lib.listToAttrs ( + lib.imap (index: workerName: { + name = workerName; + value = { + worker_listeners = [ + { + type = "http"; + port = 9200 + index + 1; + resources = [ + { + names = [ "metrics" ]; + compress = false; + } + ]; + } + ]; + }; + }) config.monitoring.synapse.workerNames + ) + ); + + services.prometheus.scrapeConfigs = ( + (lib.map (interval: { + job_name = "synapse-main-${toString interval}s"; + scrape_interval = "${toString interval}s"; + static_configs = [ { targets = [ "localhost:9200" ]; } ]; + }) cfg.prometheusScrapeIntervals) + ++ lib.flatten ( + lib.imap ( + index: workerName: + lib.map (interval: { + job_name = "synapse-${workerName}-${toString interval}s"; + scrape_interval = "${toString interval}s"; + static_configs = [ { targets = [ "localhost:${toString (9200 + index + 1)}" ]; } ]; + }) cfg.prometheusScrapeIntervals + ) config.monitoring.synapse.workerNames + ) + ); + + services.grafana.provision.dashboards.settings = { + apiVersion = 1; + providers = [ + { + name = "matrix-synapse"; + orgId = 1; + type = "file"; + options = { + path = builtins.fetchurl { + url = "https://raw.githubusercontent.com/element-hq/synapse/master/contrib/grafana/synapse.json"; + sha256 = "07qlr0waw9phmyd38bv22bn5v303w3397b89l44l3lzwhpnhs16s"; + }; + }; + } + ]; + }; + + }; +} diff --git a/modules/monitoring/system.nix b/modules/monitoring/system.nix new file mode 100644 index 0000000..cc1ec66 --- /dev/null +++ b/modules/monitoring/system.nix @@ -0,0 +1,75 @@ +{ lib, config, ... }: +let + cfg = config.monitoring; +in +{ + config = lib.mkIf (cfg.monitorAll) { + services.prometheus.exporters.node = { + enable = true; + port = 9100; + enabledCollectors = [ + #"logind" #too slow + "systemd" + "processes" + "interrupts" + # Testing: + "buddyinfo" + "cgroups" + "ksmd" + "lnstat" + "mountstats" + "network_route" + #"perf" # requires sysctl change + "qdisc" + "sysctl" + "softirqs" + "tcpstat" + ]; + disabledCollectors = [ + "textfile" + "xfs" + "zfs" + "selinux" + "cpufreq" + "btrfs" + "powersupplyclass" + "mdadm" + "tapestats" + "fibrechannel" + "cpu_vulnerabilities" + "watchdog" + "thermal_zone" + "logind" + "nfs" + "nfsd" + "infiniband" + ]; + }; + + services.prometheus.scrapeConfigs = ( + lib.map (interval: { + job_name = "node-${toString interval}s"; + scrape_interval = "${toString interval}s"; + static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; } ]; + }) cfg.prometheusScrapeIntervals + ); + + services.grafana.provision.dashboards.settings = { + apiVersion = 1; + providers = [ + { + name = "default"; + orgId = 1; + type = "file"; + options = { + path = builtins.fetchurl { + url = "https://grafana.com/api/dashboards/1860/revisions/37/download"; + sha256 = "0qza4j8lywrj08bqbww52dgh2p2b9rkhq5p313g72i57lrlkacfl"; + }; + }; + } + ]; + }; + + }; +} diff --git a/modules/users/Rory.client.nix b/modules/users/Rory.client.nix index a9a3925..fb6ad88 100755 --- a/modules/users/Rory.client.nix +++ b/modules/users/Rory.client.nix @@ -13,6 +13,8 @@ helvum vesktop pavucontrol + wf-recorder + waypipe ]; home-manager.users.Rory = { -- cgit 1.4.1