Some synapse work, expose nheko-git and mtxclient-git packages

8 files changed, 714 insertions, 6 deletions

diff --git a/host/Rory-nginx/services/matrix/synapse/synapse-main.nix b/host/Rory-nginx/services/matrix/synapse/synapse-main.nix
index e8f24d5..9c12e75 100755
--- a/host/Rory-nginx/services/matrix/synapse/synapse-main.nix
+++ b/host/Rory-nginx/services/matrix/synapse/synapse-main.nix
@@ -12,7 +12,7 @@
     nginxVirtualHostName = "matrix.rory.gay";
     enableWorkers = true;
 
-    federationSenders = 8; #8
+    federationSenders = 8; #16
     pushers = 1;
     mediaRepoWorkers = 2; #4
     clientReaders = 2; #4
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/account_data-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/account_data-stream-writer.nix
new file mode 100644
index 0000000..200e7c9
--- /dev/null
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/account_data-stream-writer.nix
@@ -0,0 +1,118 @@
+{ config, lib, ... }:
+
+let
+  cfg = config.services.matrix-synapse;
+  dbGroup = "medium";
+  streamWriterType = "account_data";
+  workers = lib.range 0 (cfg.accountDataStreamWriters - 1);
+  workerName = "account_data_stream_writer";
+  workerRoutes = {
+    client = [ ];
+    federation = [ ];
+    media = [ ];
+  };
+in
+let
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+in
+{
+  config = lib.mkIf (cfg.accountDataStreamWriters > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
+    services.matrix-synapse = {
+      settings = {
+        instance_map = lib.listToAttrs (
+          lib.map (index: {
+            name = "${workerName}-${toString index}";
+            value = {
+              path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+            };
+          }) workers
+        );
+
+        stream_writers.${streamWriterType} = lib.map (index: "${workerName}-${toString index}") workers;
+      };
+
+      workers = lib.listToAttrs (
+        lib.map (index: {
+          name = "${workerName}-${toString index}";
+          value = {
+            worker_app = "synapse.app.generic_worker";
+            worker_listeners =
+              [
+                {
+                  type = "http";
+                  path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+                  resources = [
+                    {
+                      names = [ "replication" ];
+                      compress = false;
+                    }
+                  ];
+                }
+              ]
+              ++ lib.map (type: {
+                type = "http";
+                path = "/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                mode = "666";
+                resources = [
+                  {
+                    names = [ type ];
+                    compress = false;
+                  }
+                ];
+              }) enabledResources;
+            database = (
+              import ../../db.nix {
+                inherit dbGroup;
+                workerName = "${workerName}-${toString index}";
+              }
+            );
+          };
+        }) workers
+      );
+    };
+
+    services.nginx = {
+      upstreams = lib.listToAttrs (
+        lib.map (type: {
+          name = "${workerName}-${type}";
+          value = {
+            extraConfig = ''
+              keepalive 32;
+              least_conn;
+            '';
+            servers = lib.listToAttrs (
+              lib.map (index: {
+                name = "unix:/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                value = {
+                  max_fails = 0;
+                };
+              }) workers
+            );
+          };
+        }) enabledResources
+      );
+
+      virtualHosts."${cfg.nginxVirtualHostName}".locations = lib.listToAttrs (
+        lib.flatten (
+          lib.forEach enabledResources (
+            type:
+            lib.map (route: {
+              name = route;
+              value = {
+                proxyPass = "http://${workerName}-${type}";
+                extraConfig = ''
+                  proxy_http_version 1.1;
+                  proxy_set_header Connection "";
+                '';
+              };
+            }) workerRoutes.${type}
+          )
+        )
+      );
+    };
+  };
+}
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/presence-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/presence-stream-writer.nix
index e906b28..84da90d 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/presence-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/presence-stream-writer.nix
@@ -33,11 +33,6 @@ in
         );
 
         stream_writers.${streamWriterType} = lib.map (index: "${workerName}-${toString index}") workers;
-        stream_writers.typing = lib.map (index: "${workerName}-${toString index}") workers;
-        stream_writers.to_device = lib.map (index: "${workerName}-${toString index}") workers;
-        stream_writers.account_data = lib.map (index: "${workerName}-${toString index}") workers;
-        stream_writers.receipts = lib.map (index: "${workerName}-${toString index}") workers;
-        stream_writers.push_rules = lib.map (index: "${workerName}-${toString index}") workers;
       };
 
       workers = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/push_rules-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/push_rules-stream-writer.nix
new file mode 100644
index 0000000..f4a6acc
--- /dev/null
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/push_rules-stream-writer.nix
@@ -0,0 +1,118 @@
+{ config, lib, ... }:
+
+let
+  cfg = config.services.matrix-synapse;
+  dbGroup = "medium";
+  streamWriterType = "push_rules";
+  workers = lib.range 0 (cfg.pushRuleStreamWriters - 1);
+  workerName = "push_rule_stream_writer";
+  workerRoutes = {
+    client = [ ];
+    federation = [ ];
+    media = [ ];
+  };
+in
+let
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+in
+{
+  config = lib.mkIf (cfg.pushRuleStreamWriters > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
+    services.matrix-synapse = {
+      settings = {
+        instance_map = lib.listToAttrs (
+          lib.map (index: {
+            name = "${workerName}-${toString index}";
+            value = {
+              path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+            };
+          }) workers
+        );
+
+        stream_writers.${streamWriterType} = lib.map (index: "${workerName}-${toString index}") workers;
+      };
+
+      workers = lib.listToAttrs (
+        lib.map (index: {
+          name = "${workerName}-${toString index}";
+          value = {
+            worker_app = "synapse.app.generic_worker";
+            worker_listeners =
+              [
+                {
+                  type = "http";
+                  path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+                  resources = [
+                    {
+                      names = [ "replication" ];
+                      compress = false;
+                    }
+                  ];
+                }
+              ]
+              ++ lib.map (type: {
+                type = "http";
+                path = "/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                mode = "666";
+                resources = [
+                  {
+                    names = [ type ];
+                    compress = false;
+                  }
+                ];
+              }) enabledResources;
+            database = (
+              import ../../db.nix {
+                inherit dbGroup;
+                workerName = "${workerName}-${toString index}";
+              }
+            );
+          };
+        }) workers
+      );
+    };
+
+    services.nginx = {
+      upstreams = lib.listToAttrs (
+        lib.map (type: {
+          name = "${workerName}-${type}";
+          value = {
+            extraConfig = ''
+              keepalive 32;
+              least_conn;
+            '';
+            servers = lib.listToAttrs (
+              lib.map (index: {
+                name = "unix:/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                value = {
+                  max_fails = 0;
+                };
+              }) workers
+            );
+          };
+        }) enabledResources
+      );
+
+      virtualHosts."${cfg.nginxVirtualHostName}".locations = lib.listToAttrs (
+        lib.flatten (
+          lib.forEach enabledResources (
+            type:
+            lib.map (route: {
+              name = route;
+              value = {
+                proxyPass = "http://${workerName}-${type}";
+                extraConfig = ''
+                  proxy_http_version 1.1;
+                  proxy_set_header Connection "";
+                '';
+              };
+            }) workerRoutes.${type}
+          )
+        )
+      );
+    };
+  };
+}
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/receipts-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/receipts-stream-writer.nix
new file mode 100644
index 0000000..91583d9
--- /dev/null
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/receipts-stream-writer.nix
@@ -0,0 +1,118 @@
+{ config, lib, ... }:
+
+let
+  cfg = config.services.matrix-synapse;
+  dbGroup = "medium";
+  streamWriterType = "receipts";
+  workers = lib.range 0 (cfg.receiptStreamWriters - 1);
+  workerName = "receipts_stream_writer";
+  workerRoutes = {
+    client = [ ];
+    federation = [ ];
+    media = [ ];
+  };
+in
+let
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+in
+{
+  config = lib.mkIf (cfg.receiptStreamWriters > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
+    services.matrix-synapse = {
+      settings = {
+        instance_map = lib.listToAttrs (
+          lib.map (index: {
+            name = "${workerName}-${toString index}";
+            value = {
+              path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+            };
+          }) workers
+        );
+
+        stream_writers.${streamWriterType} = lib.map (index: "${workerName}-${toString index}") workers;
+      };
+
+      workers = lib.listToAttrs (
+        lib.map (index: {
+          name = "${workerName}-${toString index}";
+          value = {
+            worker_app = "synapse.app.generic_worker";
+            worker_listeners =
+              [
+                {
+                  type = "http";
+                  path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+                  resources = [
+                    {
+                      names = [ "replication" ];
+                      compress = false;
+                    }
+                  ];
+                }
+              ]
+              ++ lib.map (type: {
+                type = "http";
+                path = "/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                mode = "666";
+                resources = [
+                  {
+                    names = [ type ];
+                    compress = false;
+                  }
+                ];
+              }) enabledResources;
+            database = (
+              import ../../db.nix {
+                inherit dbGroup;
+                workerName = "${workerName}-${toString index}";
+              }
+            );
+          };
+        }) workers
+      );
+    };
+
+    services.nginx = {
+      upstreams = lib.listToAttrs (
+        lib.map (type: {
+          name = "${workerName}-${type}";
+          value = {
+            extraConfig = ''
+              keepalive 32;
+              least_conn;
+            '';
+            servers = lib.listToAttrs (
+              lib.map (index: {
+                name = "unix:/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                value = {
+                  max_fails = 0;
+                };
+              }) workers
+            );
+          };
+        }) enabledResources
+      );
+
+      virtualHosts."${cfg.nginxVirtualHostName}".locations = lib.listToAttrs (
+        lib.flatten (
+          lib.forEach enabledResources (
+            type:
+            lib.map (route: {
+              name = route;
+              value = {
+                proxyPass = "http://${workerName}-${type}";
+                extraConfig = ''
+                  proxy_http_version 1.1;
+                  proxy_set_header Connection "";
+                '';
+              };
+            }) workerRoutes.${type}
+          )
+        )
+      );
+    };
+  };
+}
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/shared-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/shared-stream-writer.nix
new file mode 100644
index 0000000..3da4276
--- /dev/null
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/shared-stream-writer.nix
@@ -0,0 +1,123 @@
+{ config, lib, ... }:
+
+let
+  cfg = config.services.matrix-synapse;
+  dbGroup = "medium";
+  streamWriterType = "shared"; # unused, from template
+  workers = lib.range 0 (cfg.presenceStreamWriters - 1);
+  workerName = "shared_stream_writer";
+  workerRoutes = {
+    client = [ ];
+    federation = [ ];
+    media = [ ];
+  };
+in
+let
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+in
+{
+  config = lib.mkIf (cfg.presenceStreamWriters > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
+    services.matrix-synapse = {
+      settings = {
+        instance_map = lib.listToAttrs (
+          lib.map (index: {
+            name = "${workerName}-${toString index}";
+            value = {
+              path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+            };
+          }) workers
+        );
+
+        stream_writers.account_data = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.events = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.typing = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.to_device = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.receipts = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.push_rules = lib.map (index: "${workerName}-${toString index}") workers;
+      };
+
+      workers = lib.listToAttrs (
+        lib.map (index: {
+          name = "${workerName}-${toString index}";
+          value = {
+            worker_app = "synapse.app.generic_worker";
+            worker_listeners =
+              [
+                {
+                  type = "http";
+                  path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+                  resources = [
+                    {
+                      names = [ "replication" ];
+                      compress = false;
+                    }
+                  ];
+                }
+              ]
+              ++ lib.map (type: {
+                type = "http";
+                path = "/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                mode = "666";
+                resources = [
+                  {
+                    names = [ type ];
+                    compress = false;
+                  }
+                ];
+              }) enabledResources;
+            database = (
+              import ../../db.nix {
+                inherit dbGroup;
+                workerName = "${workerName}-${toString index}";
+              }
+            );
+          };
+        }) workers
+      );
+    };
+
+    services.nginx = {
+      upstreams = lib.listToAttrs (
+        lib.map (type: {
+          name = "${workerName}-${type}";
+          value = {
+            extraConfig = ''
+              keepalive 32;
+              least_conn;
+            '';
+            servers = lib.listToAttrs (
+              lib.map (index: {
+                name = "unix:/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                value = {
+                  max_fails = 0;
+                };
+              }) workers
+            );
+          };
+        }) enabledResources
+      );
+
+      virtualHosts."${cfg.nginxVirtualHostName}".locations = lib.listToAttrs (
+        lib.flatten (
+          lib.forEach enabledResources (
+            type:
+            lib.map (route: {
+              name = route;
+              value = {
+                proxyPass = "http://${workerName}-${type}";
+                extraConfig = ''
+                  proxy_http_version 1.1;
+                  proxy_set_header Connection "";
+                '';
+              };
+            }) workerRoutes.${type}
+          )
+        )
+      );
+    };
+  };
+}
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/to_device-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/to_device-stream-writer.nix
new file mode 100644
index 0000000..e24c8a4
--- /dev/null
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/to_device-stream-writer.nix
@@ -0,0 +1,118 @@
+{ config, lib, ... }:
+
+let
+  cfg = config.services.matrix-synapse;
+  dbGroup = "medium";
+  streamWriterType = "to_device";
+  workers = lib.range 0 (cfg.toDeviceStreamWriters - 1);
+  workerName = "to_device_stream_writer";
+  workerRoutes = {
+    client = [ ];
+    federation = [ ];
+    media = [ ];
+  };
+in
+let
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+in
+{
+  config = lib.mkIf (cfg.toDeviceStreamWriters > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
+    services.matrix-synapse = {
+      settings = {
+        instance_map = lib.listToAttrs (
+          lib.map (index: {
+            name = "${workerName}-${toString index}";
+            value = {
+              path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+            };
+          }) workers
+        );
+
+        stream_writers.${streamWriterType} = lib.map (index: "${workerName}-${toString index}") workers;
+      };
+
+      workers = lib.listToAttrs (
+        lib.map (index: {
+          name = "${workerName}-${toString index}";
+          value = {
+            worker_app = "synapse.app.generic_worker";
+            worker_listeners =
+              [
+                {
+                  type = "http";
+                  path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+                  resources = [
+                    {
+                      names = [ "replication" ];
+                      compress = false;
+                    }
+                  ];
+                }
+              ]
+              ++ lib.map (type: {
+                type = "http";
+                path = "/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                mode = "666";
+                resources = [
+                  {
+                    names = [ type ];
+                    compress = false;
+                  }
+                ];
+              }) enabledResources;
+            database = (
+              import ../../db.nix {
+                inherit dbGroup;
+                workerName = "${workerName}-${toString index}";
+              }
+            );
+          };
+        }) workers
+      );
+    };
+
+    services.nginx = {
+      upstreams = lib.listToAttrs (
+        lib.map (type: {
+          name = "${workerName}-${type}";
+          value = {
+            extraConfig = ''
+              keepalive 32;
+              least_conn;
+            '';
+            servers = lib.listToAttrs (
+              lib.map (index: {
+                name = "unix:/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                value = {
+                  max_fails = 0;
+                };
+              }) workers
+            );
+          };
+        }) enabledResources
+      );
+
+      virtualHosts."${cfg.nginxVirtualHostName}".locations = lib.listToAttrs (
+        lib.flatten (
+          lib.forEach enabledResources (
+            type:
+            lib.map (route: {
+              name = route;
+              value = {
+                proxyPass = "http://${workerName}-${type}";
+                extraConfig = ''
+                  proxy_http_version 1.1;
+                  proxy_set_header Connection "";
+                '';
+              };
+            }) workerRoutes.${type}
+          )
+        )
+      );
+    };
+  };
+}
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/typing-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/typing-stream-writer.nix
new file mode 100644
index 0000000..80e79a9
--- /dev/null
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/typing-stream-writer.nix
@@ -0,0 +1,118 @@
+{ config, lib, ... }:
+
+let
+  cfg = config.services.matrix-synapse;
+  dbGroup = "medium";
+  streamWriterType = "typing";
+  workers = lib.range 0 (cfg.typingStreamWriters - 1);
+  workerName = "typing_stream_writer";
+  workerRoutes = {
+    client = [ ];
+    federation = [ ];
+    media = [ ];
+  };
+in
+let
+  enabledResources =
+    lib.optionals (lib.length workerRoutes.client > 0) [ "client" ]
+    ++ lib.optionals (lib.length workerRoutes.federation > 0) [ "federation" ]
+    ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
+in
+{
+  config = lib.mkIf (cfg.typingStreamWriters > 0) {
+    monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
+    services.matrix-synapse = {
+      settings = {
+        instance_map = lib.listToAttrs (
+          lib.map (index: {
+            name = "${workerName}-${toString index}";
+            value = {
+              path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+            };
+          }) workers
+        );
+
+        stream_writers.${streamWriterType} = lib.map (index: "${workerName}-${toString index}") workers;
+      };
+
+      workers = lib.listToAttrs (
+        lib.map (index: {
+          name = "${workerName}-${toString index}";
+          value = {
+            worker_app = "synapse.app.generic_worker";
+            worker_listeners =
+              [
+                {
+                  type = "http";
+                  path = "/run/matrix-synapse/${workerName}-${toString index}.sock";
+                  resources = [
+                    {
+                      names = [ "replication" ];
+                      compress = false;
+                    }
+                  ];
+                }
+              ]
+              ++ lib.map (type: {
+                type = "http";
+                path = "/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                mode = "666";
+                resources = [
+                  {
+                    names = [ type ];
+                    compress = false;
+                  }
+                ];
+              }) enabledResources;
+            database = (
+              import ../../db.nix {
+                inherit dbGroup;
+                workerName = "${workerName}-${toString index}";
+              }
+            );
+          };
+        }) workers
+      );
+    };
+
+    services.nginx = {
+      upstreams = lib.listToAttrs (
+        lib.map (type: {
+          name = "${workerName}-${type}";
+          value = {
+            extraConfig = ''
+              keepalive 32;
+              least_conn;
+            '';
+            servers = lib.listToAttrs (
+              lib.map (index: {
+                name = "unix:/run/matrix-synapse/${workerName}-${type}-${toString index}.sock";
+                value = {
+                  max_fails = 0;
+                };
+              }) workers
+            );
+          };
+        }) enabledResources
+      );
+
+      virtualHosts."${cfg.nginxVirtualHostName}".locations = lib.listToAttrs (
+        lib.flatten (
+          lib.forEach enabledResources (
+            type:
+            lib.map (route: {
+              name = route;
+              value = {
+                proxyPass = "http://${workerName}-${type}";
+                extraConfig = ''
+                  proxy_http_version 1.1;
+                  proxy_set_header Connection "";
+                '';
+              };
+            }) workerRoutes.${type}
+          )
+        )
+      );
+    };
+  };
+}