Synapse worker changes

12 files changed, 175 insertions, 45 deletions

diff --git a/host/Rory-nginx/services/matrix/synapse/db.nix b/host/Rory-nginx/services/matrix/synapse/db.nix
index 24c714e..77d9773 100644
--- a/host/Rory-nginx/services/matrix/synapse/db.nix
+++ b/host/Rory-nginx/services/matrix/synapse/db.nix
@@ -32,5 +32,19 @@
         10
       else
         throw "synapse/db.nix: Invalid dbGroup: ${if dbGroup == null then "null" else dbGroup}";
+        
+    # cp_reconnect - default=True - https://github.com/element-hq/synapse/blob/develop/synapse/storage/database.py#L129
+    # cp_noisy - default=False - https://docs.twisted.org/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__ - info logs during operation
+    # check_same_thread - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/config/database.py#L65 - can this even be set?
   };
+  
+  
+  # synchronous_commit - default=True - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L56 
+  # statement_timeout - default=60 * 60 * 1000 ms - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L63
+  # allow_unsafe_locale - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L99
+  # allow_outdated_version - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L92 - needs source link
+  # txn_limit - default=0 - https://github.com/element-hq/synapse/blob/develop/synapse/storage/database.py#L564
+  
+  statement_timeout = 24 * 60 * 60 * 1000; #24 hours, good for bg jobs
+  txn_limit = 500; #maybe dropping old data from pg caches helps?
 }
diff --git a/host/Rory-nginx/services/matrix/synapse/ratelimits.nix b/host/Rory-nginx/services/matrix/synapse/ratelimits.nix
index 4f6b7db..ffce1cc 100644
--- a/host/Rory-nginx/services/matrix/synapse/ratelimits.nix
+++ b/host/Rory-nginx/services/matrix/synapse/ratelimits.nix
@@ -1,22 +1,15 @@
 {
+  # messages
   rc_message = {
-    per_second = 1000;
-    burst_count = 1000;
+    per_second = 1000000;
+    burst_count = 1000000;
   };
-  rc_login = {
-    address = {
-      per_second = 1000;
-      burst_count = 1000;
-    };
-    account = {
-      per_second = 1000;
-      burst_count = 1000;
-    };
-    failed_attempts = {
-      per_second = 0.1;
-      burst_count = 3;
-    };
+  rc_admin_redaction = {
+    per_second = 10000000;
+    burst_count = 10000000;
   };
+
+  # room joins
   rc_joins = {
     local = {
       per_second = 1000;
@@ -31,6 +24,8 @@
     per_second = 1000;
     burst_count = 1000;
   };
+
+  # room invites
   rc_invites = {
     per_room = {
       per_second = 1000;
@@ -45,6 +40,12 @@
       burst_count = 1000;
     };
   };
+  rc_third_party_invite = {
+    per_second = 1000;
+    burst_count = 1000;
+  };
+
+  # federation
   rc_federation = {
     window_size = 10;
     sleep_limit = 1000;
@@ -53,4 +54,32 @@
     concurrent = 100;
   };
   federation_rr_transactions_per_room_per_second = 1;
+
+  # media
+  rc_media_create = {
+    per_second = 1000;
+    burst_count = 1000;
+  };
+  remote_media_download_burst_count = "512G";
+  remote_media_download_per_second = "512G";
+
+  # authentication
+  rc_login = {
+    address = {
+      per_second = 1000;
+      burst_count = 1000;
+    };
+    account = {
+      per_second = 1000;
+      burst_count = 1000;
+    };
+    failed_attempts = {
+      per_second = 0.1;
+      burst_count = 3;
+    };
+  };
+  rc_3pid_validation = {
+    per_second = 1000;
+    burst_count = 1000;
+  };
 }
diff --git a/host/Rory-nginx/services/matrix/synapse/synapse-main.nix b/host/Rory-nginx/services/matrix/synapse/synapse-main.nix
index 7387d46..3bc185b 100755
--- a/host/Rory-nginx/services/matrix/synapse/synapse-main.nix
+++ b/host/Rory-nginx/services/matrix/synapse/synapse-main.nix
@@ -12,27 +12,47 @@
     nginxVirtualHostName = "matrix.rory.gay";
     enableWorkers = true;
 
-    federationSenders = 8; #16
+    federationSenders = 8; # 16
     pushers = 1;
-    mediaRepoWorkers = 2; #4
-    clientReaders = 2; #4
-    syncWorkers = 2; #4
-    authWorkers = 1;
+    mediaRepoWorkers = 2; # 4
+    clientReaders = 2; # 4
+    syncWorkers = 2; # 4
+    authWorkers = 0;
 
-    federationReaders = 4; #8
-    federationInboundWorkers = 4; #8
+    federationReaders = 4; # 8
+    federationInboundWorkers = 4; # 8
 
     enableAppserviceWorker = true;
     enableBackgroundWorker = true;
     enableUserDirWorker = true;
 
-    eventStreamWriters = 4; #8
+    accountDataStreamWriters = 1;
+    eventStreamWriters = 4; # 8
     presenceStreamWriters = 1;
+    pushRuleStreamWriters = 1;
+    receiptStreamWriters = 1;
+    toDeviceStreamWriters = 1;
+    typingStreamWriters = 1;
+
+    #untested:
+    #sharedStreamWriters = 1;
 
     # https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html
     settings = {
       server_name = "rory.gay";
 
+      use_frozen_dicts = true;
+      user_agent_suffix = " (rory.gay)";
+
+      # look into later: replication_torture_level - https://github.com/element-hq/synapse/blob/develop/synapse/config/server.py#L560
+      # limit_remote_rooms ???
+      # cleanup_extremities_with_dummy_events - default=True
+      # dummy_devents_treshold - default=10 - required forward extremities to send dummy event
+      # enable_ephemeral_messages - default=False - ???
+      # rooms_to_exclude_from_sync - default=[] - room ids...
+      # third_party_event_rules - https://github.com/element-hq/synapse/blob/develop/synapse/config/third_party_event_rules.py - ???
+      # default_power_level_content_override - default=None - https://github.com/element-hq/synapse/blob/develop/synapse/config/room.py#L73
+
       enable_registration = true;
       registration_requires_token = true;
 
@@ -42,11 +62,16 @@
       allow_device_name_lookup_over_federation = true;
 
       federation = {
-        client_timeout = "60s";
+        client_timeout = "30s"; # default=60s
         max_short_retries = 12;
         max_short_retry_delay = "5s";
         max_long_retries = 5;
         max_long_retry_delay = "30s";
+        
+        # rapid retry, small increments
+        destination_min_retry_interval = "5m"; # default=10m 
+        destination_max_retry_interval = "12h"; #default=7d
+        destination_retry_multiplier = 1.2; #default=2
       };
 
       registration_shared_secret_path = "/var/lib/matrix-synapse/registration_shared_secret.txt";
@@ -54,9 +79,7 @@
       listeners = [
         {
           port = 8008;
-          bind_addresses = [
-            "127.0.0.1"
-          ];
+          bind_addresses = [ "127.0.0.1" ];
           type = "http";
           tls = false;
           x_forwarded = true;
@@ -81,12 +104,10 @@
           ];
         }
       ];
-      dynamic_thumbnails = true;
       presence = {
-        enable = true;
+        enablee = true;
         update_interval = 60;
       };
-      url_preview_enabled = true;
       database = (
         import ./db.nix {
           workerName = "main";
@@ -98,8 +119,25 @@
         "/var/lib/matrix-synapse/modas-registration.yaml"
       ];
 
+      #region Media
+      max_upload_size = "512M";
+
+      max_avatar_size = "512M";
       max_image_pixels = "250M";
 
+      max_pending_media_uploads = 512;
+      dynamic_thumbnails = true;
+
+      prevent_media_downloads_from = [
+        # none, give me all the media
+      ];
+      enable_authenticated_media = false;
+
+      url_preview_enabled = true;
+      max_spider_size = "50M";
+
+      #endregion
+
       ui_auth = {
         session_timeout = "1m";
       };
@@ -124,6 +162,8 @@
         "msc3026_enabled" = true; # Busy presence
         "msc3266_enabled" = true; # Room summary API
         "msc3916_authenticated_media_enabled" = true; # Authenticated media
+        "msc3823_account_suspension" = true; # Account suspension
+        "msc4151_enabled" = true; # Report room API (CS-API)
       };
 
       redis = {
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix b/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix
index a361390..e52010c 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/media-repo.nix
@@ -80,9 +80,6 @@ in
               }
             );
             enable_media_repo = true;
-            max_upload_size = "512M";
-            remote_media_download_burst_count = "512G";
-            remote_media_download_per_second = "512G";
             rc_federation = {
               window_size = 1;
               sleep_limit = 1000;
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/module.nix b/host/Rory-nginx/services/matrix/synapse/workers/module.nix
index 87e014e..a02540c 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/module.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/module.nix
@@ -14,7 +14,7 @@ in
     ./single/appservice.nix
     ./single/background.nix
     ./single/user-dir.nix
-    
+
     ./auth.nix
     ./client-reader.nix
     ./event-creator.nix
@@ -25,7 +25,15 @@ in
     ./pusher.nix
     ./sync.nix
 
+    ./stream-writers/account_data-stream-writer.nix
     ./stream-writers/event-stream-writer.nix
+    ./stream-writers/presence-stream-writer.nix
+    ./stream-writers/push_rule-stream-writer.nix
+    ./stream-writers/receipt-stream-writer.nix
+    ./stream-writers/to_device-stream-writer.nix
+    ./stream-writers/typing-stream-writer.nix
+
+    ./stream-writers/shared-stream-writer.nix
   ];
   options.services.matrix-synapse = {
     enableWorkers = lib.mkEnableOption "Enable dedicated workers";
@@ -49,9 +57,11 @@ in
     typingStreamWriters = mkIntOption "Number of typing stream writers";
     toDeviceStreamWriters = mkIntOption "Number of to_device stream writers";
     accountDataStreamWriters = mkIntOption "Number of account data stream writers";
-    receiptsStreamWriters = mkIntOption "Number of read receipt stream writers";
+    receiptStreamWriters = mkIntOption "Number of read receipt stream writers";
     presenceStreamWriters = mkIntOption "Number of presence stream writers";
     pushRuleStreamWriters = mkIntOption "Number of push rule stream writers";
+    
+    sharedStreamWriters = mkIntOption "Number of shared stream writers";
 
     nginxVirtualHostName = lib.mkOption {
       type = lib.types.str;
@@ -66,6 +76,39 @@ in
         assertion = cfg.enableWorkers -> cfg.nginxVirtualHostName != null;
         message = "nginxVirtualHostName must be set when enableWorkers is true";
       }
+
+
+      # Stream types and count limitations: https://github.com/element-hq/synapse/blob/develop/synapse/config/workers.py#L344
+      {
+        assertion = cfg.typingStreamWriters <= 1;
+        message = "Only one typing stream writer is supported";
+      }
+      {
+        assertion = cfg.toDeviceStreamWriters <= 1;
+        message = "Only one to_device stream writer is supported";
+      }
+      {
+        assertion = cfg.accountDataStreamWriters <= 1;
+        message = "Only one account data stream writer is supported";
+      }
+      # This may be outdated in the documentation...?
+      #{
+      #  assertion = cfg.receiptStreamWriters <= 1;
+      #  message = "Only one receipt stream writer is supported";
+      #}
+      {
+        assertion = cfg.presenceStreamWriters <= 1;
+        message = "Only one presence stream writer is supported";
+      }
+      {
+        assertion = cfg.pushRuleStreamWriters <= 1;
+        message = "Only one push rule stream writer is supported";
+      }
+      
+      {
+        assertion = cfg.sharedStreamWriters <= 1;
+        message = "Only one shared stream writer is supported";
+      }
     ];
   };
 }
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/account_data-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/account_data-stream-writer.nix
index 200e7c9..dff6e36 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/account_data-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/account_data-stream-writer.nix
@@ -7,7 +7,10 @@ let
   workers = lib.range 0 (cfg.accountDataStreamWriters - 1);
   workerName = "account_data_stream_writer";
   workerRoutes = {
-    client = [ ];
+    client = [
+      "^/_matrix/client/(r0|v3|unstable)/.*/tags"
+      "^/_matrix/client/(r0|v3|unstable)/.*/account_data"
+    ];
     federation = [ ];
     media = [ ];
   };
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/presence-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/presence-stream-writer.nix
index 84da90d..69d4813 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/presence-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/presence-stream-writer.nix
@@ -7,7 +7,7 @@ let
   workers = lib.range 0 (cfg.presenceStreamWriters - 1);
   workerName = "presence_stream_writer";
   workerRoutes = {
-    client = [ ];
+    client = [ "^/_matrix/client/(api/v1|r0|v3|unstable)/presence/" ];
     federation = [ ];
     media = [ ];
   };
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/push_rules-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/push_rule-stream-writer.nix
index f4a6acc..fbd0327 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/push_rules-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/push_rule-stream-writer.nix
@@ -7,7 +7,7 @@ let
   workers = lib.range 0 (cfg.pushRuleStreamWriters - 1);
   workerName = "push_rule_stream_writer";
   workerRoutes = {
-    client = [ ];
+    client = [ "^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/" ];
     federation = [ ];
     media = [ ];
   };
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/receipts-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/receipt-stream-writer.nix
index 91583d9..da4e3a2 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/receipts-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/receipt-stream-writer.nix
@@ -7,7 +7,10 @@ let
   workers = lib.range 0 (cfg.receiptStreamWriters - 1);
   workerName = "receipts_stream_writer";
   workerRoutes = {
-    client = [ ];
+    client = [
+      "^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt"
+      "^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers"
+    ];
     federation = [ ];
     media = [ ];
   };
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/shared-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/shared-stream-writer.nix
index 3da4276..5fd0bd0 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/shared-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/shared-stream-writer.nix
@@ -19,7 +19,7 @@ let
     ++ lib.optionals (lib.length workerRoutes.media > 0) [ "media" ];
 in
 {
-  config = lib.mkIf (cfg.presenceStreamWriters > 0) {
+  config = lib.mkIf (cfg.sharedStreamWriters > 0) {
     monitoring.synapse.workerNames = lib.map (index: "${workerName}-${toString index}") workers;
     services.matrix-synapse = {
       settings = {
@@ -34,10 +34,11 @@ in
 
         stream_writers.account_data = lib.map (index: "${workerName}-${toString index}") workers;
         stream_writers.events = lib.map (index: "${workerName}-${toString index}") workers;
-        stream_writers.typing = lib.map (index: "${workerName}-${toString index}") workers;
-        stream_writers.to_device = lib.map (index: "${workerName}-${toString index}") workers;
-        stream_writers.receipts = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.presence = lib.map (index: "${workerName}-${toString index}") workers;
         stream_writers.push_rules = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.receipts = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.to_device = lib.map (index: "${workerName}-${toString index}") workers;
+        stream_writers.typing = lib.map (index: "${workerName}-${toString index}") workers;
       };
 
       workers = lib.listToAttrs (
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/to_device-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/to_device-stream-writer.nix
index e24c8a4..47c2c0a 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/to_device-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/to_device-stream-writer.nix
@@ -7,7 +7,7 @@ let
   workers = lib.range 0 (cfg.toDeviceStreamWriters - 1);
   workerName = "to_device_stream_writer";
   workerRoutes = {
-    client = [ ];
+    client = [ "^/_matrix/client/(r0|v3|unstable)/sendToDevice/" ];
     federation = [ ];
     media = [ ];
   };
diff --git a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/typing-stream-writer.nix b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/typing-stream-writer.nix
index 80e79a9..3986619 100644
--- a/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/typing-stream-writer.nix
+++ b/host/Rory-nginx/services/matrix/synapse/workers/stream-writers/typing-stream-writer.nix
@@ -7,7 +7,7 @@ let
   workers = lib.range 0 (cfg.typingStreamWriters - 1);
   workerName = "typing_stream_writer";
   workerRoutes = {
-    client = [ ];
+    client = [ "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing" ];
     federation = [ ];
     media = [ ];
   };