Monitoring?

24 files changed, 466 insertions, 28 deletions

diff --git a/flake.lock b/flake.lock
index d00ae07..d86d743 100644
--- a/flake.lock
+++ b/flake.lock
@@ -404,11 +404,11 @@
       },
       "locked": {
         "host": "gitlab.computer.surgery",
-        "lastModified": 1724088992,
-        "narHash": "sha256-e45kLOGHA6PDDhYzq3dV4SsYf+hr2+n5SDnta9NrN14=",
+        "lastModified": 1724527899,
+        "narHash": "sha256-aLCrE4J9BTSoqWmiLQqQngt1ZNgsDZ2bOZgPRUbV2L4=",
         "owner": "matrix",
         "repo": "grapevine-fork",
-        "rev": "c355e2ad394a3c925acf24c87854e0898d0e328d",
+        "rev": "14afa1357e199e542835fde621e99052e7d12f2b",
         "type": "gitlab"
       },
       "original": {
@@ -572,11 +572,11 @@
     },
     "nixpkgs-master": {
       "locked": {
-        "lastModified": 1724475946,
-        "narHash": "sha256-rEseutXa9VP4HFiEChUX0cw7hbilCkMq/RBeKT3XPV4=",
+        "lastModified": 1724591773,
+        "narHash": "sha256-3G7MkSEWUIigBmpd5djDtfZfx7SfPj+crUw7gdrgO3k=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "77e2fa9398204bb8cda108ce1e56c282be7fdb9e",
+        "rev": "9e2335ac963ae6863194f516b57b01febd499e84",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index d34ccc4..bcef2ce 100755
--- a/flake.nix
+++ b/flake.nix
@@ -17,9 +17,7 @@
     };
 
     # Draupnir module/package
-    nixpkgs-Draupnir = {
-      url = "github:TheArcaneBrony/nixpkgs/master";
-    };
+    nixpkgs-Draupnir.url = "github:TheArcaneBrony/nixpkgs/master";
 
     # Base modules
     home-manager.url = "github:nix-community/home-manager/master";
@@ -79,6 +77,17 @@
                     matrix-synapse-unwrapped = inputs.nixpkgs-master.legacyPackages.${pkgs.stdenv.hostPlatform.system}.matrix-synapse-unwrapped;
                     draupnir = inputs.nixpkgs-Draupnir.legacyPackages.${pkgs.stdenv.hostPlatform.system}.draupnir;
                   })
+
+                  #                  // matrix-nio
+                  (final: prev: {
+                    python312 = prev.python312.override {
+                      packageOverrides = python-self: python-super: {
+                        matrix-nio = python-super.matrix-nio.overridePythonAttrs (old: {
+                          pythonRelaxDeps = [ "aiohttp-socks" ];
+                        });
+                      };
+                    };
+                  })
                 ];
               }
             )
diff --git a/host/Rory-desktop/configuration.nix b/host/Rory-desktop/configuration.nix
index 3e21734..aa09659 100644
--- a/host/Rory-desktop/configuration.nix
+++ b/host/Rory-desktop/configuration.nix
@@ -24,8 +24,8 @@ args@{
     ./optional/gui/x11.nix
     ./optional/gui/wayland.nix
 
-    ./printing.nix
-    #./ollama.nix
+    #./printing.nix
+#    ./ollama.nix
   ];
 
   boot = {
@@ -230,7 +230,7 @@ args@{
   nixpkgs = {
     config = {
       allowUnfree = true;
-      permittedInsecurePackages = [ 
+      permittedInsecurePackages = [
         "electron-25.9.0"
         "olm-3.2.16"
       ];
@@ -262,5 +262,17 @@ args@{
   virtualisation.libvirtd.enable = true;
   programs.virt-manager.enable = true;
 
+  monitoring = {
+    monitorAll = true;
+    localPrometheus = true;
+    exposePrometheus = true;
+    localGrafana = true;
+    exposeGrafana = true;
+    nginxHost = "monitoring.localhost";
+    nginxSsl = false;
+  };
+  networking.hosts."127.0.0.1" = builtins.attrNames config.services.nginx.virtualHosts;
+  
+
   system.stateVersion = "22.11"; # DO NOT EDIT!
 }
diff --git a/host/Rory-nginx/configuration.nix b/host/Rory-nginx/configuration.nix
index ebcffd3..456e15c 100755
--- a/host/Rory-nginx/configuration.nix
+++ b/host/Rory-nginx/configuration.nix
@@ -1,5 +1,6 @@
 {
   pkgs,
+  config,
   lib,
   grapevine,
   conduit,
@@ -51,6 +52,18 @@
     };
   };
 
+  monitoring = {
+    monitorAll = true;
+    localPrometheus = true;
+    exposePrometheus = true;
+    localGrafana = true;
+    exposeGrafana = true;
+    nginxHost = "monitoring.rory.gay";
+    nginxSsl = true;
+  };
+
+  nixpkgs.config.permittedInsecurePackages = [ "olm-3.2.16" ];
+
   services.irqbalance.enable = true;
 
   environment.memoryAllocator.provider = "jemalloc";
@@ -60,9 +73,7 @@
   #  conduit = grapevine;
   #};
 
-  containers."matrixunittests" = import ./services/containers/matrixunittests/container.nix {
-    inherit pkgs lib grapevine;
-  };
+  containers."matrixunittests" = import ./services/containers/matrixunittests/container.nix { inherit pkgs lib grapevine; };
 
   containers."matrixunittests-conduit" = import ./services/containers/matrixunittests-conduit/container.nix {
     inherit pkgs lib;
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/auth.nix b/host/Rory-nginx/services/matrix/synapse/workers/auth.nix
index 50d52c6..3c8d1e9 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/auth.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/auth.nix
@@ -23,10 +23,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.authWorkers > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix b/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix
index 249084a..60c48a2 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix
@@ -46,10 +46,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.clientReaders > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix b/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix
index c841b93..115ce51 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix
@@ -19,10 +19,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.eventCreators > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/federation-inbound.nix b/host/Rory-nginx/services/matrix/synapse/workers/federation-inbound.nix
index 83ceec7..effaa69 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/federation-inbound.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/federation-inbound.nix
@@ -12,10 +12,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.federationInboundWorkers > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/federation-reader.nix b/host/Rory-nginx/services/matrix/synapse/workers/federation-reader.nix
index 6a16fb6..5b3d4bf 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/federation-reader.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/federation-reader.nix
@@ -36,10 +36,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.federationReaders > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/federation-sender.nix b/host/Rory-nginx/services/matrix/synapse/workers/federation-sender.nix
index cd482f9..07af127 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/federation-sender.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/federation-sender.nix
@@ -11,10 +11,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.federationSenders > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix b/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix
index 62f3e9c..a361390 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix
@@ -21,10 +21,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.mediaRepoWorkers > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/pusher.nix b/host/Rory-nginx/services/matrix/synapse/workers/pusher.nix
index 7388824..edf1632 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/pusher.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/pusher.nix
@@ -11,10 +11,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.pushers > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/single/appservice.nix b/host/Rory-nginx/services/matrix/synapse/workers/single/appservice.nix
index 315a322..119fd04 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/single/appservice.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/single/appservice.nix
@@ -11,10 +11,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf cfg.enableAppserviceWorker {
+    monitoring.synapse.workerNames = [ workerName ];
     services.matrix-synapse = {
       settings = {
         instance_map = {
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/single/background.nix b/host/Rory-nginx/services/matrix/synapse/workers/single/background.nix
index b667e19..37fde10 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/single/background.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/single/background.nix
@@ -13,10 +13,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf cfg.enableBackgroundWorker {
+    monitoring.synapse.workerNames = [ workerName ];
     services.matrix-synapse = {
       settings = {
         instance_map = {
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/single/user-dir.nix b/host/Rory-nginx/services/matrix/synapse/workers/single/user-dir.nix
index 89c6aa4..f26f3ec 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/single/user-dir.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/single/user-dir.nix
@@ -16,10 +16,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf cfg.enableUserDirWorker {
+    monitoring.synapse.workerNames = [ workerName ];
     services.matrix-synapse = {
       settings = {
         instance_map = {
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/event-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/event-stream-writer.nix
index 35bd4e9..5395aea 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/event-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/event-stream-writer.nix
@@ -13,10 +13,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.eventStreamWriters > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/sync.nix b/host/Rory-nginx/services/matrix/synapse/workers/sync.nix
index 636f521..6892847 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/sync.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/sync.nix
@@ -17,10 +17,14 @@ let
   };
 in
 let
-  enabledResources = lib.optionals (lib.length workerRoutes.client > 0) [ "client" ] ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ] ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
   config = lib.mkIf (cfg.syncWorkers > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
         instance_map = lib.listToAttrs (
diff --git a/modules/base.nix b/modules/base.nix
index bb122a8..4cf2aff 100755
--- a/modules/base.nix
+++ b/modules/base.nix
@@ -5,6 +5,7 @@
     ./packages/vim.nix
     ./users/Rory.nix
     ./extra-substituters.nix
+    ./monitoring/module.nix
   ];
 
   boot = {
diff --git a/modules/monitoring/crutches/synapse.nix b/modules/monitoring/crutches/synapse.nix
new file mode 100644
index 0000000..3145d11
--- /dev/null
+++ b/modules/monitoring/crutches/synapse.nix
@@ -0,0 +1,11 @@
+{lib, ...}:
+
+{
+  options.monitoring.synapse = {
+    workerNames = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [];
+      description = "Synapse worker names";
+    };
+  };
+}
diff --git a/modules/monitoring/module.nix b/modules/monitoring/module.nix
new file mode 100644
index 0000000..60e396d
--- /dev/null
+++ b/modules/monitoring/module.nix
@@ -0,0 +1,106 @@
+{ lib, config, ... }:
+let
+  mkStringOption =
+    description:
+    lib.mkOption {
+      type = lib.types.str;
+      default = null;
+      description = description;
+    };
+  cfg = config.monitoring;
+in
+{
+  imports = [
+    # crutches due to nix limitations:
+    ./crutches/synapse.nix
+    
+    # modules
+    ./system.nix
+    ./postgres.nix
+    ./synapse.nix
+  ];
+  options.monitoring = {
+    monitorAll = lib.mkEnableOption "Monitor all services";
+    localPrometheus = lib.mkEnableOption "Local Prometheus";
+    exposePrometheus = lib.mkEnableOption "Expose Prometheus";
+    localGrafana = lib.mkEnableOption "Local Grafana";
+    exposeGrafana = lib.mkEnableOption "Expose Grafana";
+
+    nginxHost = mkStringOption "The virtual host name";
+    nginxSsl = lib.mkEnableOption "Enable SSL for Nginx";
+    prometheusScrapeIntervals = lib.mkOption {
+      type = lib.types.listOf lib.types.int;
+      default = [
+        1
+        2
+        5
+        15
+      ];
+      description = "Scrape intervals for Prometheus";
+    };
+  };
+
+  config = lib.mkIf (cfg.monitorAll) {
+    services = {
+      prometheus = lib.mkIf (cfg.localPrometheus) {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+      grafana = lib.mkIf (cfg.localGrafana) {
+        enable = true;
+        settings = {
+          server = {
+            domain = cfg.nginxHost;
+            enable_gzip = true;
+            protocol = "socket";
+            socket_mode = "0666";
+          };
+        };
+        provision = {
+          datasources.settings = {
+            apiVersion = 1;
+            #            datasources = [
+            #              {
+            #                name = "Prometheus";
+            #                type = "prometheus";
+            #                access = "proxy";
+            #                url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+            #                isDefault = true;
+            #              }
+            #            ];
+
+            datasources = lib.map (interval: {
+              name = "Prometheus-${toString interval}";
+              type = "prometheus";
+              access = "proxy";
+              url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+              #isDefault = true;
+              jsonData.timeInterval = "${toString interval}s";
+            }) cfg.prometheusScrapeIntervals;
+          };
+        };
+      };
+      nginx.virtualHosts = {
+        "${cfg.nginxHost}" = {
+          enableACME = cfg.nginxSsl;
+          addSSL = cfg.nginxSsl;
+          http3 = cfg.nginxSsl;
+          http3_hq = cfg.nginxSsl;
+          kTLS = cfg.nginxSsl;
+          locations = {
+            "/" = if cfg.exposeGrafana then { proxyPass = "http://unix:${config.services.grafana.settings.server.socket}"; } else { return = "200 'OK'"; };
+          };
+        };
+
+        "prometheus.${cfg.nginxHost}" = lib.mkIf (cfg.exposePrometheus) {
+          enableACME = cfg.nginxSsl;
+          addSSL = cfg.nginxSsl;
+          http3 = cfg.nginxSsl;
+          http3_hq = cfg.nginxSsl;
+          kTLS = cfg.nginxSsl;
+          locations."/".proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}";
+        };
+      };
+    };
+  };
+}
diff --git a/modules/monitoring/postgres.nix b/modules/monitoring/postgres.nix
new file mode 100644
index 0000000..bbb55e6
--- /dev/null
+++ b/modules/monitoring/postgres.nix
@@ -0,0 +1,48 @@
+{ lib, config, ... }:
+let
+  cfg = config.monitoring;
+in
+{
+  config = lib.mkIf (cfg.monitorAll && config.services.postgresql.enable) {
+    services.prometheus.exporters.postgres = {
+      enable = true;
+      extraFlags = [
+        "--collector.database_wraparound"
+        "--collector.long_running_transactions"
+        "--collector.postmaster"
+        "--collector.process_idle"
+        "--collector.stat_activity_autovacuum"
+        "--collector.stat_statements"
+        #"--collector.stat_wal_receiver" #we dont have WAL receivers
+        "--collector.statio_user_indexes"
+        "--collector.xlog_location"
+      ];
+    };
+
+    services.prometheus.scrapeConfigs = (
+      lib.map (interval: {
+        job_name = "postgres-${toString interval}s";
+        scrape_interval = "${toString interval}s";
+        static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ]; } ];
+      }) cfg.prometheusScrapeIntervals
+    );
+
+    services.grafana.provision.dashboards.settings = {
+      apiVersion = 1;
+      providers = [
+        {
+          name = "14114-postgres-overview";
+          orgId = 1;
+          type = "file";
+          options = {
+            path = builtins.fetchurl {
+              url = "https://grafana.com/api/dashboards/14114/revisions/1/download";
+              sha256 = "0qza4j8lywrj08bqbww52dgh2p2b9rkhq5p313g72i57lrlkacfl";
+            };
+          };
+        }
+      ];
+    };
+
+  };
+}
diff --git a/modules/monitoring/synapse.nix b/modules/monitoring/synapse.nix
new file mode 100644
index 0000000..12c5562
--- /dev/null
+++ b/modules/monitoring/synapse.nix
@@ -0,0 +1,111 @@
+{ lib, config, ... }:
+let
+  cfg = config.monitoring;
+in
+{
+  config = lib.mkIf (cfg.monitorAll && config.services.matrix-synapse.enable) {
+    services.matrix-synapse.settings.listeners = [
+      {
+        type = "http";
+        port = 9200;
+        resources = [
+          {
+            names = [ "metrics" ];
+            compress = false;
+          }
+        ];
+      }
+    ];
+
+    #services.matrix-synapse.workers = (
+    #  let
+    #    workerNames = config.services.matrix-synapse.workers;
+
+    #    hasMetricsListener =
+    #      (workerName:
+    #      lib.any (
+    #        listener: listener.type == "http" && (lib.any (resourceName: resourceName == "metrics") listener.resources)
+    #      ) config.services.matrix-synapse.workers.${workerName}.worker_listeners);
+
+    #    workerNamesWithoutMetrics = lib.traceVal lib.filter (workerName: (!hasMetricsListener workerName)) workerNames;
+    #  in
+    #  lib.listToAttrs (
+    #    lib.imap (index: workerName: {
+    #      name = workerName;
+    #      value = {
+    #        worker_listeners = [
+    #          {
+    #            type = "http";
+    #            port = 9200 + index;
+    #            resources = [
+    #              {
+    #                names = [ "metrics" ];
+    #                compress = false;
+    #              }
+    #            ];
+    #          }
+    #        ];
+    #      };
+    #    }) workerNamesWithoutMetrics
+    #  )
+    #);
+
+    services.matrix-synapse.workers = (
+      lib.listToAttrs (
+        lib.imap (index: workerName: {
+          name = workerName;
+          value = {
+            worker_listeners = [
+              {
+                type = "http";
+                port = 9200 + index + 1;
+                resources = [
+                  {
+                    names = [ "metrics" ];
+                    compress = false;
+                  }
+                ];
+              }
+            ];
+          };
+        }) config.monitoring.synapse.workerNames
+      )
+    );
+
+    services.prometheus.scrapeConfigs = (
+      (lib.map (interval: {
+        job_name = "synapse-main-${toString interval}s";
+        scrape_interval = "${toString interval}s";
+        static_configs = [ { targets = [ "localhost:9200" ]; } ];
+      }) cfg.prometheusScrapeIntervals)
+      ++ lib.flatten (
+        lib.imap (
+          index: workerName:
+          lib.map (interval: {
+            job_name = "synapse-${workerName}-${toString interval}s";
+            scrape_interval = "${toString interval}s";
+            static_configs = [ { targets = [ "localhost:${toString (9200 + index + 1)}" ]; } ];
+          }) cfg.prometheusScrapeIntervals
+        ) config.monitoring.synapse.workerNames
+      )
+    );
+
+    services.grafana.provision.dashboards.settings = {
+      apiVersion = 1;
+      providers = [
+        {
+          name = "matrix-synapse";
+          orgId = 1;
+          type = "file";
+          options = {
+            path = builtins.fetchurl {
+              url = "https://raw.githubusercontent.com/element-hq/synapse/master/contrib/grafana/synapse.json";
+              sha256 = "07qlr0waw9phmyd38bv22bn5v303w3397b89l44l3lzwhpnhs16s";
+            };
+          };
+        }
+      ];
+    };
+
+  };
+}
diff --git a/modules/monitoring/system.nix b/modules/monitoring/system.nix
new file mode 100644
index 0000000..cc1ec66
--- /dev/null
+++ b/modules/monitoring/system.nix
@@ -0,0 +1,75 @@
+{ lib, config, ... }:
+let
+  cfg = config.monitoring;
+in
+{
+  config = lib.mkIf (cfg.monitorAll) {
+    services.prometheus.exporters.node = {
+      enable = true;
+      port = 9100;
+      enabledCollectors = [
+        #"logind" #too slow
+        "systemd"
+        "processes"
+        "interrupts"
+        # Testing:
+        "buddyinfo"
+        "cgroups"
+        "ksmd"
+        "lnstat"
+        "mountstats"
+        "network_route"
+        #"perf" # requires sysctl change
+        "qdisc"
+        "sysctl"
+        "softirqs"
+        "tcpstat"
+      ];
+      disabledCollectors = [
+        "textfile"
+        "xfs"
+        "zfs"
+        "selinux"
+        "cpufreq"
+        "btrfs"
+        "powersupplyclass"
+        "mdadm"
+        "tapestats"
+        "fibrechannel"
+        "cpu_vulnerabilities"
+        "watchdog"
+        "thermal_zone"
+        "logind"
+        "nfs"
+        "nfsd"
+        "infiniband"
+      ];
+    };
+
+    services.prometheus.scrapeConfigs = (
+        lib.map (interval: {
+            job_name = "node-${toString interval}s";
+            scrape_interval = "${toString interval}s";
+            static_configs = [ { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; } ];
+        }) cfg.prometheusScrapeIntervals
+    );
+
+    services.grafana.provision.dashboards.settings = {
+      apiVersion = 1;
+      providers = [
+        {
+          name = "default";
+          orgId = 1;
+          type = "file";
+          options = {
+            path = builtins.fetchurl {
+              url = "https://grafana.com/api/dashboards/1860/revisions/37/download";
+              sha256 = "0qza4j8lywrj08bqbww52dgh2p2b9rkhq5p313g72i57lrlkacfl";
+            };
+          };
+        }
+      ];
+    };
+
+  };
+}
diff --git a/modules/users/Rory.client.nix b/modules/users/Rory.client.nix
index a9a3925..fb6ad88 100755
--- a/modules/users/Rory.client.nix
+++ b/modules/users/Rory.client.nix
@@ -13,6 +13,8 @@
     helvum
     vesktop
     pavucontrol
+    wf-recorder
+    waypipe
   ];
 
   home-manager.users.Rory = {