From 7d84cf83297b3ec97afa6abc401f03426f63d9c5 Mon Sep 17 00:00:00 2001 From: Rory& Date: Tue, 22 Jul 2025 09:29:10 +0200 Subject: Add synapse container, nixfmt --- flake.nix | 21 +- host/Arc/configuration.nix | 14 +- host/Arc/optional/gui/wayland.nix | 8 +- host/Rory-NTFS/configuration.nix | 12 +- host/Rory-desktop/configuration.nix | 9 +- host/Rory-desktop/optional/gui/hyprland.nix | 2 +- host/Rory-desktop/services/libvirt.nix | 2 +- host/Rory-desktop/services/nginx/hse.localhost.nix | 32 ++-- .../services/nginx/matrix.opensuse.localhost.nix | 32 ++-- .../services/nginx/synapse.localhost.nix | 2 +- host/Rory-desktop/services/rabbitmq.nix | 2 +- host/Rory-desktop/services/steam.nix | 2 +- host/Rory-desktop/services/waydroid.nix | 2 +- host/Rory-laptop/configuration.nix | 36 ++-- host/Rory-laptop/nginx.nix | 2 +- host/Rory-laptop/nginx/hse.localhost.nix | 32 ++-- host/Rory-laptop/optional/gui/wayland.nix | 4 +- host/Rory-nginx/services/mastodon.nix | 6 +- .../services/matrix/synapse/ratelimits.nix | 4 +- .../matrix/synapse/workers/client-reader.nix | 6 +- .../matrix/synapse/workers/event-creator.nix | 6 +- host/Rory-nginx/services/nginx/rory.gay/mru.nix | 4 +- host/Rory-ovh/configuration.nix | 19 ++ host/Rory-ovh/services/cgit.nix | 7 +- .../containers/draupnir-ansible/container.nix | 20 +- .../services/containers/draupnir-cme/container.nix | 12 +- .../containers/draupnir-fedora/container.nix | 20 +- .../services/containers/syntest1/container.nix | 73 +++++++ .../Rory-ovh/services/containers/syntest1/root.nix | 16 ++ .../containers/syntest1/services/postgres.nix | 99 ++++++++++ .../syntest1/services/synapse/caches.nix | 25 +++ .../containers/syntest1/services/synapse/db.nix | 49 +++++ .../syntest1/services/synapse/ratelimits.nix | 97 ++++++++++ .../syntest1/services/synapse/synapse-main.nix | 212 +++++++++++++++++++++ .../services/containers/syntest2/container.nix | 63 ++++++ .../Rory-ovh/services/containers/syntest2/root.nix | 16 ++ .../containers/syntest2/services/postgres.nix | 99 ++++++++++ .../syntest2/services/synapse/caches.nix | 25 +++ .../containers/syntest2/services/synapse/db.nix | 49 +++++ .../syntest2/services/synapse/ratelimits.nix | 97 ++++++++++ .../syntest2/services/synapse/synapse-main.nix | 211 ++++++++++++++++++++ host/Rory-ovh/services/email/maddy.nix | 6 +- host/Rory-ovh/services/mastodon.nix | 6 +- host/Rory-ovh/services/matrix/synapse/caches.nix | 2 +- .../services/matrix/synapse/synapse-main.nix | 7 +- host/Rory-ovh/services/nginx/nginx.nix | 19 +- .../services/nginx/rory.gay/api.safensound.nix | 2 +- host/Rory-ovh/services/nginx/rory.gay/cgit.nix | 2 +- .../nginx/rory.gay/conduit.matrixunittests.nix | 2 +- .../services/nginx/rory.gay/demo.safensound.nix | 6 +- host/Rory-ovh/services/nginx/rory.gay/ec.nix | 2 +- .../services/nginx/rory.gay/libmatrix-fed-test.nix | 2 +- .../services/nginx/rory.gay/matrix-bak.nix | 2 +- host/Rory-ovh/services/nginx/rory.gay/matrix.nix | 6 +- .../services/nginx/rory.gay/matrixunittests.nix | 2 +- host/Rory-ovh/services/nginx/rory.gay/mru.nix | 6 +- host/Rory-ovh/services/nginx/rory.gay/root.nix | 2 +- .../services/nginx/rory.gay/safensound.nix | 6 +- host/Rory-ovh/services/nginx/rory.gay/stream.nix | 38 ++-- host/Rory-ovh/services/nginx/rory.gay/syntest1.nix | 68 +++++++ host/Rory-ovh/services/nginx/rory.gay/syntest2.nix | 68 +++++++ host/RoryNix/configuration.nix | 5 +- host/uISO/development.nix | 6 +- host/uISO/iso-root.nix | 18 +- host/uISO/overlays/grub-overlay.nix | 32 ++-- host/uISO/overlays/systemd-overlay.nix | 2 +- host/uISO/pkgs/resize.nix | 7 +- modules/base.nix | 4 +- modules/software-templates/steam.nix | 7 +- .../software-templates/synapse-workers/generic.nix | 8 +- .../synapse-workers/generic/auth.nix | 2 +- .../synapse-workers/generic/client-reader.nix | 2 +- .../synapse-workers/generic/event-creator.nix | 2 +- .../synapse-workers/generic/federation-inbound.nix | 2 +- .../synapse-workers/generic/federation-reader.nix | 2 +- .../synapse-workers/generic/federation-sender.nix | 10 +- .../synapse-workers/generic/pusher.nix | 2 +- .../synapse-workers/generic/sync.nix | 2 +- modules/software-templates/synapse-workers/lib.nix | 6 +- .../synapse-workers/single/appservice.nix | 2 +- .../stream-writers/account_data-stream-writer.nix | 7 +- .../stream-writers/event-stream-writer.nix | 4 +- modules/users/Rory.nix | 2 +- packages/matrix-spec.nix | 16 +- .../patches/0001-Fast-auth-links.patch | 100 ++++++++++ ...-ignore-rejected-events-in-delayed_events.patch | 31 --- ...h-logging-to-room-summary-over-federation.patch | 77 -------- ...-ignore-rejected-events-in-delayed_events.patch | 31 +++ ...h-logging-to-room-summary-over-federation.patch | 77 ++++++++ ...-entire-room-if-accessibility-check-fails.patch | 28 --- ...-entire-room-if-accessibility-check-fails.patch | 28 +++ .../0004-Log-policy-server-rejected-events.patch | 30 --- .../0005-Log-policy-server-rejected-events.patch | 30 +++ ...-Use-parse_boolean-for-unredacted-content.patch | 29 --- .../0006-Expose-tombstone-in-room-admin-api.patch | 114 ----------- ...-Use-parse_boolean-for-unredacted-content.patch | 29 +++ .../0007-Expose-tombstone-in-room-admin-api.patch | 114 +++++++++++ ...recheck-messages-pagination-data-if-a-bac.patch | 204 -------------------- ...nation-with-large-gaps-of-rejected-events.patch | 50 ----- ...recheck-messages-pagination-data-if-a-bac.patch | 204 ++++++++++++++++++++ .../patches/0009-Fix-nix-flake.patch | 189 ------------------ ...nation-with-large-gaps-of-rejected-events.patch | 50 +++++ .../0010-Fix-gitignore-to-ignore-.venv.patch | 24 --- .../patches/0010-Fix-nix-flake.patch | 189 ++++++++++++++++++ .../0011-Fix-gitignore-to-ignore-.venv.patch | 24 +++ .../patches/synapse-fast-links.patch | 87 --------- 106 files changed, 2463 insertions(+), 1129 deletions(-) create mode 100644 host/Rory-ovh/services/containers/syntest1/container.nix create mode 100644 host/Rory-ovh/services/containers/syntest1/root.nix create mode 100755 host/Rory-ovh/services/containers/syntest1/services/postgres.nix create mode 100644 host/Rory-ovh/services/containers/syntest1/services/synapse/caches.nix create mode 100644 host/Rory-ovh/services/containers/syntest1/services/synapse/db.nix create mode 100644 host/Rory-ovh/services/containers/syntest1/services/synapse/ratelimits.nix create mode 100755 host/Rory-ovh/services/containers/syntest1/services/synapse/synapse-main.nix create mode 100644 host/Rory-ovh/services/containers/syntest2/container.nix create mode 100644 host/Rory-ovh/services/containers/syntest2/root.nix create mode 100755 host/Rory-ovh/services/containers/syntest2/services/postgres.nix create mode 100644 host/Rory-ovh/services/containers/syntest2/services/synapse/caches.nix create mode 100644 host/Rory-ovh/services/containers/syntest2/services/synapse/db.nix create mode 100644 host/Rory-ovh/services/containers/syntest2/services/synapse/ratelimits.nix create mode 100755 host/Rory-ovh/services/containers/syntest2/services/synapse/synapse-main.nix create mode 100755 host/Rory-ovh/services/nginx/rory.gay/syntest1.nix create mode 100755 host/Rory-ovh/services/nginx/rory.gay/syntest2.nix create mode 100644 packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0001-Hotfix-ignore-rejected-events-in-delayed_events.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0002-Add-too-much-logging-to-room-summary-over-federation.patch create mode 100644 packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch create mode 100644 packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0003-Log-entire-room-if-accessibility-check-fails.patch create mode 100644 packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0004-Log-policy-server-rejected-events.patch create mode 100644 packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0005-Use-parse_boolean-for-unredacted-content.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0006-Expose-tombstone-in-room-admin-api.patch create mode 100644 packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch create mode 100644 packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0007-fix-Always-recheck-messages-pagination-data-if-a-bac.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0008-Fix-pagination-with-large-gaps-of-rejected-events.patch create mode 100644 packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0009-Fix-nix-flake.patch create mode 100644 packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0010-Fix-gitignore-to-ignore-.venv.patch create mode 100644 packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch create mode 100644 packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch delete mode 100644 packages/overlays/matrix-synapse/patches/synapse-fast-links.patch diff --git a/flake.nix b/flake.nix index a24a96c..675a614 100755 --- a/flake.nix +++ b/flake.nix @@ -18,9 +18,9 @@ url = "github:NixOS/nixpkgs/nixos-unstable"; }; -# nixpkgs-stable = { -# url = "github:NixOS/nixpkgs/nixos-24.11"; -# }; + # nixpkgs-stable = { + # url = "github:NixOS/nixpkgs/nixos-24.11"; + # }; nixpkgs-master = { url = "github:NixOS/nixpkgs/master"; @@ -34,7 +34,7 @@ # Draupnir module/package #nixpkgs-DraupnirPkg.url = "github:r-ryantm/nixpkgs/auto-update/draupnir"; #nixpkgs-DraupnirPkg.url = "github:NixOS/nixpkgs/master"; -# nixpkgs-keydb.url = "github:NixOS/nixpkgs?rev=e0464e47880a69896f0fb1810f00e0de469f770a"; + # nixpkgs-keydb.url = "github:NixOS/nixpkgs?rev=e0464e47880a69896f0fb1810f00e0de469f770a"; #MatrixContentFilter.url = "git+file:/home/Rory/git/matrix/MatrixContentFilter?submodules=1"; # Base modules @@ -146,7 +146,7 @@ safeNSound.nixosModules.default ( - { pkgs, lib, ... }: + { pkgs, lib, ... }: { disabledModules = [ "services/matrix/synapse.nix" ]; imports = [ @@ -177,6 +177,8 @@ inherit cgit-magenta; inherit (inputs) draupnirSrc; + inherit (inputs) nixpkgs-master; + matrix-synapse-unwrapped-patched = self.packages.x86_64-linux.matrix-synapse-unwrapped-patched; }; }; @@ -422,9 +424,9 @@ packages.redpanda-connect = (pkgs.callPackage ./packages/redpanda-connect/default.nix { }); packages.matrix-synapse-unwrapped-patched = pkgs-master.matrix-synapse-unwrapped.overrideAttrs (oldAttrs: rec { - patches = (if oldAttrs ? patches then oldAttrs.patches else []) ++ pkgs.lib.map ( - path: ./packages/overlays/matrix-synapse/patches/${path} - ) (builtins.attrNames (builtins.readDir ./packages/overlays/matrix-synapse/patches)); + patches = + (if oldAttrs ? patches then oldAttrs.patches else [ ]) + ++ pkgs.lib.map (path: ./packages/overlays/matrix-synapse/patches/${path}) (builtins.attrNames (builtins.readDir ./packages/overlays/matrix-synapse/patches)); cargoDeps = pkgs.rustPlatform.fetchCargoVendor { inherit (oldAttrs) src; @@ -433,6 +435,9 @@ hash = "sha256-BfLj+cqS6zpX2qLb+Rur4cy7CyPH2KzdXaTXpNK20DM="; }; }); + packages.matrix-synapse-patched = pkgs-master.matrix-synapse.override { + matrix-synapse-unwrapped = self.packages.${pkgs.stdenv.hostPlatform.system}.matrix-synapse-unwrapped-patched; + }; } ); } diff --git a/host/Arc/configuration.nix b/host/Arc/configuration.nix index b97a8fa..f12348c 100644 --- a/host/Arc/configuration.nix +++ b/host/Arc/configuration.nix @@ -84,7 +84,7 @@ args@{ jack.enable = true; alsa.enable = true; }; - + desktopManager.plasma6.enable = true; }; @@ -171,9 +171,15 @@ args@{ nerd-fonts.jetbrains-mono noto-fonts-monochrome-emoji ]; - fontconfig.defaultFonts.monospace = with pkgs; [ "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" ]; - fontconfig.defaultFonts.sansSerif = with pkgs; [ "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" ]; - fontconfig.defaultFonts.serif = with pkgs; [ "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" ]; + fontconfig.defaultFonts.monospace = with pkgs; [ + "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" + ]; + fontconfig.defaultFonts.sansSerif = with pkgs; [ + "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" + ]; + fontconfig.defaultFonts.serif = with pkgs; [ + "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" + ]; fontconfig.defaultFonts.emoji = with pkgs; [ "freefont" ]; enableDefaultPackages = lib.mkForce false; enableGhostscriptFonts = lib.mkForce false; diff --git a/host/Arc/optional/gui/wayland.nix b/host/Arc/optional/gui/wayland.nix index 281c72d..0d8682b 100644 --- a/host/Arc/optional/gui/wayland.nix +++ b/host/Arc/optional/gui/wayland.nix @@ -8,7 +8,7 @@ { #programs.sway = { # enable = true; -# + # # wrapperFeatures.gtk = true; # extraSessionCommands = '' # # -- Wayland fixes @@ -21,7 +21,7 @@ # # use this if they aren't displayed properly: # export _JAVA_AWT_WM_NONREPARENTING=1 # ''; -# + # # extraPackages = with pkgs; [ # swaybg # #swayidle @@ -41,7 +41,7 @@ }; programs.hyprlock.enable = true; - environment.systemPackages = with pkgs;[ + environment.systemPackages = with pkgs; [ wmenu waybar @@ -52,7 +52,7 @@ keepassxc networkmanagerapplet ]; - + #environment.etc."hyprPlugins" = { # text = '' # plugin = ${hy3.packages.${pkgs.stdenv.hostPlatform.system}.hy3}/lib/libhy3.so diff --git a/host/Rory-NTFS/configuration.nix b/host/Rory-NTFS/configuration.nix index 98e28de..47d4497 100644 --- a/host/Rory-NTFS/configuration.nix +++ b/host/Rory-NTFS/configuration.nix @@ -52,7 +52,7 @@ # allowedUDPPorts = [ ... ]; }; - #interfaces.enp34s0.ipv4.addresses = [ { + #interfaces.enp34s0.ipv4.addresses = [ { # address = "192.168.0.3"; # prefixLength = 24; #} ]; @@ -170,7 +170,7 @@ extraPortals = with pkgs; [ #xdg-desktop-portal-gtk xdg-desktop-portal-xapp -# (callPackage ../../packages/xdg-desktop-portal-gtk.nix { }) + # (callPackage ../../packages/xdg-desktop-portal-gtk.nix { }) ]; config = { common = { @@ -190,8 +190,12 @@ nerd-fonts.jetbrains-mono noto-fonts-monochrome-emoji ]; - fontconfig.defaultFonts.monospace = [ "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" ]; - fontconfig.defaultFonts.sansSerif = [ "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" ]; + fontconfig.defaultFonts.monospace = [ + "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" + ]; + fontconfig.defaultFonts.sansSerif = [ + "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" + ]; fontconfig.defaultFonts.serif = [ "JetBrainsMonoNL Nerd Font,JetBrainsMonoNL NF:style=Regular" ]; fontconfig.defaultFonts.emoji = [ "freefont" ]; enableDefaultPackages = lib.mkForce false; diff --git a/host/Rory-desktop/configuration.nix b/host/Rory-desktop/configuration.nix index 27f5d93..814bc0f 100644 --- a/host/Rory-desktop/configuration.nix +++ b/host/Rory-desktop/configuration.nix @@ -50,7 +50,7 @@ args@{ # TODO: re-enable when USB is fixed #programs.noisetorch.enable = true; - + programs.gamemode.enable = true; users.users.Rory.extraGroups = [ "gamemode" ]; @@ -58,7 +58,7 @@ args@{ ZSH_DISABLE_COMPFIX = "true"; }; -# hardware.google-stadia-controller.enable = true; + # hardware.google-stadia-controller.enable = true; networking = { hostName = "Rory-desktop"; @@ -127,7 +127,7 @@ args@{ # - IDEs #jetbrains-toolbox - (jetbrains.plugins.addPlugins jetbrains.webstorm [ + (jetbrains.plugins.addPlugins jetbrains.webstorm [ jetbrains.plugins.github-copilot-fixed #"github-copilot" ]) @@ -136,7 +136,7 @@ args@{ #"github-copilot" "nixidea" "visual-studio-keymap" - ]) #"string-manipulation" + ]) # "string-manipulation" #jetbrains.rider #jetbrains.clion #github-copilot-intellij-agent @@ -180,7 +180,6 @@ args@{ #vesktop #discord-development - # (dwarf-fortress-packages.dwarf-fortress-full.override { enableStoneSense = true; enableFPS = true; theme = dwarf-fortress-packages.themes.spacefox; }) mangohud diff --git a/host/Rory-desktop/optional/gui/hyprland.nix b/host/Rory-desktop/optional/gui/hyprland.nix index d9dcbb2..f5b0dc2 100644 --- a/host/Rory-desktop/optional/gui/hyprland.nix +++ b/host/Rory-desktop/optional/gui/hyprland.nix @@ -12,7 +12,7 @@ }; programs.hyprlock.enable = true; - environment.systemPackages = with pkgs;[ + environment.systemPackages = with pkgs; [ wmenu waybar wl-clipboard diff --git a/host/Rory-desktop/services/libvirt.nix b/host/Rory-desktop/services/libvirt.nix index 405a73d..0724f8c 100644 --- a/host/Rory-desktop/services/libvirt.nix +++ b/host/Rory-desktop/services/libvirt.nix @@ -5,4 +5,4 @@ environment.systemPackages = with pkgs; [ virt-viewer ]; -} \ No newline at end of file +} diff --git a/host/Rory-desktop/services/nginx/hse.localhost.nix b/host/Rory-desktop/services/nginx/hse.localhost.nix index 5812b02..c3c1731 100755 --- a/host/Rory-desktop/services/nginx/hse.localhost.nix +++ b/host/Rory-desktop/services/nginx/hse.localhost.nix @@ -1,4 +1,4 @@ -{pkgs, ...}: +{ pkgs, ... }: { enableACME = false; addSSL = true; @@ -27,21 +27,21 @@ ''; }; -# locations."= /.well-known/matrix/server".extraConfig = '' -# more_set_headers 'Content-Type application/json'; -# more_set_headers 'Access-Control-Allow-Origin *'; -# return 200 '${builtins.toJSON { "m.server" = "hse.localhost:5298"; }}'; -# ''; -# locations."= /.well-known/matrix/client".extraConfig = '' -# more_set_headers 'Content-Type application/json'; -# more_set_headers 'Access-Control-Allow-Origin *'; -# return 200 '${ -# builtins.toJSON { -# "m.homeserver".base_url = "http://hse.localhost:5298"; -# "org.matrix.msc3575.proxy".url = "https://matrix.rory.gay"; -# } -# }'; -# ''; + # locations."= /.well-known/matrix/server".extraConfig = '' + # more_set_headers 'Content-Type application/json'; + # more_set_headers 'Access-Control-Allow-Origin *'; + # return 200 '${builtins.toJSON { "m.server" = "hse.localhost:5298"; }}'; + # ''; + # locations."= /.well-known/matrix/client".extraConfig = '' + # more_set_headers 'Content-Type application/json'; + # more_set_headers 'Access-Control-Allow-Origin *'; + # return 200 '${ + # builtins.toJSON { + # "m.homeserver".base_url = "http://hse.localhost:5298"; + # "org.matrix.msc3575.proxy".url = "https://matrix.rory.gay"; + # } + # }'; + # ''; locations."= /.well-known/matrix/support".extraConfig = '' more_set_headers 'Content-Type application/json'; more_set_headers 'Access-Control-Allow-Origin *'; diff --git a/host/Rory-desktop/services/nginx/matrix.opensuse.localhost.nix b/host/Rory-desktop/services/nginx/matrix.opensuse.localhost.nix index 87287a4..8683a13 100644 --- a/host/Rory-desktop/services/nginx/matrix.opensuse.localhost.nix +++ b/host/Rory-desktop/services/nginx/matrix.opensuse.localhost.nix @@ -1,4 +1,4 @@ -{pkgs, ...}: +{ pkgs, ... }: { enableACME = false; addSSL = true; @@ -27,21 +27,21 @@ ''; }; -# locations."= /.well-known/matrix/server".extraConfig = '' -# more_set_headers 'Content-Type application/json'; -# more_set_headers 'Access-Control-Allow-Origin *'; -# return 200 '${builtins.toJSON { "m.server" = "hse.localhost:5298"; }}'; -# ''; -# locations."= /.well-known/matrix/client".extraConfig = '' -# more_set_headers 'Content-Type application/json'; -# more_set_headers 'Access-Control-Allow-Origin *'; -# return 200 '${ -# builtins.toJSON { -# "m.homeserver".base_url = "http://hse.localhost:5298"; -# "org.matrix.msc3575.proxy".url = "https://matrix.rory.gay"; -# } -# }'; -# ''; + # locations."= /.well-known/matrix/server".extraConfig = '' + # more_set_headers 'Content-Type application/json'; + # more_set_headers 'Access-Control-Allow-Origin *'; + # return 200 '${builtins.toJSON { "m.server" = "hse.localhost:5298"; }}'; + # ''; + # locations."= /.well-known/matrix/client".extraConfig = '' + # more_set_headers 'Content-Type application/json'; + # more_set_headers 'Access-Control-Allow-Origin *'; + # return 200 '${ + # builtins.toJSON { + # "m.homeserver".base_url = "http://hse.localhost:5298"; + # "org.matrix.msc3575.proxy".url = "https://matrix.rory.gay"; + # } + # }'; + # ''; locations."= /.well-known/matrix/support".extraConfig = '' more_set_headers 'Content-Type application/json'; more_set_headers 'Access-Control-Allow-Origin *'; diff --git a/host/Rory-desktop/services/nginx/synapse.localhost.nix b/host/Rory-desktop/services/nginx/synapse.localhost.nix index 9f89678..766d66f 100755 --- a/host/Rory-desktop/services/nginx/synapse.localhost.nix +++ b/host/Rory-desktop/services/nginx/synapse.localhost.nix @@ -1,4 +1,4 @@ -{ pkgs }: +{ pkgs }: { enableACME = false; addSSL = true; diff --git a/host/Rory-desktop/services/rabbitmq.nix b/host/Rory-desktop/services/rabbitmq.nix index f786a70..e92eccd 100644 --- a/host/Rory-desktop/services/rabbitmq.nix +++ b/host/Rory-desktop/services/rabbitmq.nix @@ -3,4 +3,4 @@ services.rabbitmq = { enable = true; }; -} \ No newline at end of file +} diff --git a/host/Rory-desktop/services/steam.nix b/host/Rory-desktop/services/steam.nix index a01b46e..2141244 100644 --- a/host/Rory-desktop/services/steam.nix +++ b/host/Rory-desktop/services/steam.nix @@ -8,4 +8,4 @@ proton-ge-bin ]; }; -} \ No newline at end of file +} diff --git a/host/Rory-desktop/services/waydroid.nix b/host/Rory-desktop/services/waydroid.nix index 45a90ea..feb47b6 100644 --- a/host/Rory-desktop/services/waydroid.nix +++ b/host/Rory-desktop/services/waydroid.nix @@ -1,4 +1,4 @@ { ... }: { virtualisation.waydroid.enable = true; -} \ No newline at end of file +} diff --git a/host/Rory-laptop/configuration.nix b/host/Rory-laptop/configuration.nix index 894878a..5a12b5e 100644 --- a/host/Rory-laptop/configuration.nix +++ b/host/Rory-laptop/configuration.nix @@ -125,11 +125,11 @@ args@{ # - IDEs (jetbrains.plugins.addPlugins jetbrains.webstorm [ - jetbrains.plugins.github-copilot-fixed + jetbrains.plugins.github-copilot-fixed #"github-copilot" ]) (jetbrains.plugins.addPlugins jetbrains.idea-ultimate [ - jetbrains.plugins.github-copilot-fixed + jetbrains.plugins.github-copilot-fixed #"github-copilot" ]) (jetbrains.plugins.addPlugins jetbrains.clion [ @@ -209,23 +209,23 @@ args@{ systemd.services."NetworkManager-wait-online".enable = false; virtualisation.libvirtd.onBoot = "ignore"; -# systemd.services."systemd-rfkill".serviceConfig.Type = "simple"; # maybe exec? + # systemd.services."systemd-rfkill".serviceConfig.Type = "simple"; # maybe exec? systemd.services."NetworkManager".serviceConfig.Type = "exec"; # maybe exec? -# systemd.services."home-manager-Rory".serviceConfig.Type = lib.mkForce "exec"; -# systemd.services."systemd-backlight@backlight:intel_backlight".serviceConfig.Type = "exec"; -# systemd.services."wpa_supplicant".serviceConfig.Type = "exec"; -# systemd.services."libvirtd".serviceConfig.Type = lib.mkForce "exec"; -# systemd.services."nginx".serviceConfig.Type = "exec"; -# systemd.services."grafana".serviceConfig.Type = "exec"; -# systemd.services."prometheus".serviceConfig.Type = "exec"; -# systemd.services."bluetooth".serviceConfig.Type = "exec"; -# systemd.services."vmware-usbarbitrator".enable = false; -# systemd.services."vmware-authdlauncher".enable = false; -# systemd.services."waydroid-container".enable = false; -# systemd.services."NetworkManager".serviceConfig.TimeoutStartSec = 10; + # systemd.services."home-manager-Rory".serviceConfig.Type = lib.mkForce "exec"; + # systemd.services."systemd-backlight@backlight:intel_backlight".serviceConfig.Type = "exec"; + # systemd.services."wpa_supplicant".serviceConfig.Type = "exec"; + # systemd.services."libvirtd".serviceConfig.Type = lib.mkForce "exec"; + # systemd.services."nginx".serviceConfig.Type = "exec"; + # systemd.services."grafana".serviceConfig.Type = "exec"; + # systemd.services."prometheus".serviceConfig.Type = "exec"; + # systemd.services."bluetooth".serviceConfig.Type = "exec"; + # systemd.services."vmware-usbarbitrator".enable = false; + # systemd.services."vmware-authdlauncher".enable = false; + # systemd.services."waydroid-container".enable = false; + # systemd.services."NetworkManager".serviceConfig.TimeoutStartSec = 10; boot.kernelParams = [ #"quiet" - "loglevel=8" #3 + "loglevel=8" # 3 #"systemd.show_status=auto" #"rd.udev.log_level=3" #"libahci.ignore_sss=1" @@ -240,7 +240,7 @@ args@{ #boot.initrd.network.enable = false; #boot.initrd.services.lvm.enable = false; #boot.initrd.compressor = "cat"; -# + # #boot.initrd.systemd.units."systemd-backlight@.service".enable = false; #boot.initrd.systemd.units."dev-ttyS0.device".enable = false; #boot.initrd.systemd.units."dev-ttyS1.device".enable = false; @@ -255,7 +255,7 @@ args@{ #boot.initrd.availableKernelModules = [ # "nvme" #]; -# + # #services.orca.enable = true; virtualisation.waydroid.enable = true; diff --git a/host/Rory-laptop/nginx.nix b/host/Rory-laptop/nginx.nix index 0a72304..fcdcb2e 100644 --- a/host/Rory-laptop/nginx.nix +++ b/host/Rory-laptop/nginx.nix @@ -31,7 +31,7 @@ additionalModules = with pkgs.nginxModules; [ moreheaders ]; virtualHosts = { "discord.localhost" = import ./nginx/discord.localhost.nix { inherit pkgs; }; - "hse.localhost" = import ./nginx/hse.localhost.nix {inherit pkgs;}; + "hse.localhost" = import ./nginx/hse.localhost.nix { inherit pkgs; }; }; }; diff --git a/host/Rory-laptop/nginx/hse.localhost.nix b/host/Rory-laptop/nginx/hse.localhost.nix index 5812b02..c3c1731 100755 --- a/host/Rory-laptop/nginx/hse.localhost.nix +++ b/host/Rory-laptop/nginx/hse.localhost.nix @@ -1,4 +1,4 @@ -{pkgs, ...}: +{ pkgs, ... }: { enableACME = false; addSSL = true; @@ -27,21 +27,21 @@ ''; }; -# locations."= /.well-known/matrix/server".extraConfig = '' -# more_set_headers 'Content-Type application/json'; -# more_set_headers 'Access-Control-Allow-Origin *'; -# return 200 '${builtins.toJSON { "m.server" = "hse.localhost:5298"; }}'; -# ''; -# locations."= /.well-known/matrix/client".extraConfig = '' -# more_set_headers 'Content-Type application/json'; -# more_set_headers 'Access-Control-Allow-Origin *'; -# return 200 '${ -# builtins.toJSON { -# "m.homeserver".base_url = "http://hse.localhost:5298"; -# "org.matrix.msc3575.proxy".url = "https://matrix.rory.gay"; -# } -# }'; -# ''; + # locations."= /.well-known/matrix/server".extraConfig = '' + # more_set_headers 'Content-Type application/json'; + # more_set_headers 'Access-Control-Allow-Origin *'; + # return 200 '${builtins.toJSON { "m.server" = "hse.localhost:5298"; }}'; + # ''; + # locations."= /.well-known/matrix/client".extraConfig = '' + # more_set_headers 'Content-Type application/json'; + # more_set_headers 'Access-Control-Allow-Origin *'; + # return 200 '${ + # builtins.toJSON { + # "m.homeserver".base_url = "http://hse.localhost:5298"; + # "org.matrix.msc3575.proxy".url = "https://matrix.rory.gay"; + # } + # }'; + # ''; locations."= /.well-known/matrix/support".extraConfig = '' more_set_headers 'Content-Type application/json'; more_set_headers 'Access-Control-Allow-Origin *'; diff --git a/host/Rory-laptop/optional/gui/wayland.nix b/host/Rory-laptop/optional/gui/wayland.nix index ab72165..c980a85 100644 --- a/host/Rory-laptop/optional/gui/wayland.nix +++ b/host/Rory-laptop/optional/gui/wayland.nix @@ -45,11 +45,11 @@ security.rtkit.enable = true; programs.hyprlock.enable = true; - environment.systemPackages = with pkgs;[ + environment.systemPackages = with pkgs; [ wmenu kdePackages.xwaylandvideobridge ]; - + environment.etc."hyprPlugins" = { text = '' plugin = ${hy3.packages.${pkgs.stdenv.hostPlatform.system}.hy3}/lib/libhy3.so diff --git a/host/Rory-nginx/services/mastodon.nix b/host/Rory-nginx/services/mastodon.nix index 56f1808..5febe3b 100644 --- a/host/Rory-nginx/services/mastodon.nix +++ b/host/Rory-nginx/services/mastodon.nix @@ -5,8 +5,8 @@ enable = true; webProcesses = 8; webThreads = 4; - - streamingProcesses = 63; - localDomain = "rory.gay"; + + streamingProcesses = 63; + localDomain = "rory.gay"; }; } diff --git a/host/Rory-nginx/services/matrix/synapse/ratelimits.nix b/host/Rory-nginx/services/matrix/synapse/ratelimits.nix index d6d4a94..851cf50 100644 --- a/host/Rory-nginx/services/matrix/synapse/ratelimits.nix +++ b/host/Rory-nginx/services/matrix/synapse/ratelimits.nix @@ -82,13 +82,13 @@ per_second = 1000; burst_count = 1000; }; - + #presence rc_presence.per_user = { per_second = 1; burst_count = 2; }; - + #delayed events rc_delayed_event_mgmt = { per_second = 1000; diff --git a/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix b/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix index f327004..0d039c0 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/client-reader.nix @@ -114,9 +114,9 @@ in name = "${workerName}-${type}"; value = { extraConfig = '' - keepalive 32; -# least_conn; - hash $request_uri consistent; + keepalive 32; + # least_conn; + hash $request_uri consistent; ''; servers = lib.listToAttrs ( lib.map (index: { diff --git a/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix b/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix index 1c08d4b..0966573 100644 --- a/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix +++ b/host/Rory-nginx/services/matrix/synapse/workers/event-creator.nix @@ -86,9 +86,9 @@ in name = "${workerName}-${type}"; value = { extraConfig = '' - keepalive 32; -# least_conn; - hash $request_uri consistent; + keepalive 32; + # least_conn; + hash $request_uri consistent; ''; servers = lib.listToAttrs ( lib.map (index: { diff --git a/host/Rory-nginx/services/nginx/rory.gay/mru.nix b/host/Rory-nginx/services/nginx/rory.gay/mru.nix index d1e1cd7..6c7e2a1 100644 --- a/host/Rory-nginx/services/nginx/rory.gay/mru.nix +++ b/host/Rory-nginx/services/nginx/rory.gay/mru.nix @@ -1,12 +1,12 @@ { enableACME = true; addSSL = true; -# quic = true; + # quic = true; http3 = true; http3_hq = true; kTLS = true; root = "/data/nginx/html_mru"; -# reuseport = true; + # reuseport = true; extraConfig = '' brotli off; brotli_static off; diff --git a/host/Rory-ovh/configuration.nix b/host/Rory-ovh/configuration.nix index 3ded494..cd3ac32 100755 --- a/host/Rory-ovh/configuration.nix +++ b/host/Rory-ovh/configuration.nix @@ -3,6 +3,8 @@ lib, nixpkgs-Draupnir, nixpkgs-DraupnirPkg, + nixpkgs-master, + matrix-synapse-unwrapped-patched, ... }: @@ -110,6 +112,23 @@ ; }; + containers."syntest1" = import ./services/containers/syntest1/container.nix { + inherit + pkgs + lib + nixpkgs-master + matrix-synapse-unwrapped-patched + ; + }; + + containers."syntest2" = import ./services/containers/syntest2/container.nix { + inherit + pkgs + lib + nixpkgs-master + ; + }; + system.stateVersion = "22.11"; # DO NOT EDIT! environment.systemPackages = with pkgs; [ waypipe ]; diff --git a/host/Rory-ovh/services/cgit.nix b/host/Rory-ovh/services/cgit.nix index cbdc350..854dfdf 100644 --- a/host/Rory-ovh/services/cgit.nix +++ b/host/Rory-ovh/services/cgit.nix @@ -1,4 +1,9 @@ -{ pkgs, lib, cgit-magenta, ... }: +{ + pkgs, + lib, + cgit-magenta, + ... +}: let pkg = cgit-magenta.packages.${pkgs.stdenv.hostPlatform.system}.default; diff --git a/host/Rory-ovh/services/containers/draupnir-ansible/container.nix b/host/Rory-ovh/services/containers/draupnir-ansible/container.nix index 5ab1aed..ef35488 100644 --- a/host/Rory-ovh/services/containers/draupnir-ansible/container.nix +++ b/host/Rory-ovh/services/containers/draupnir-ansible/container.nix @@ -1,6 +1,6 @@ { -# nixpkgs-Draupnir, -# nixpkgs-DraupnirPkg, + # nixpkgs-Draupnir, + # nixpkgs-DraupnirPkg, ... }: @@ -8,8 +8,8 @@ privateNetwork = true; autoStart = true; specialArgs = { -# inherit nixpkgs-Draupnir; -# inherit nixpkgs-DraupnirPkg; + # inherit nixpkgs-Draupnir; + # inherit nixpkgs-DraupnirPkg; }; config = { lib, pkgs, ... }: @@ -18,13 +18,13 @@ ../shared.nix ./root.nix ./services/draupnir.nix -# "${nixpkgs-Draupnir}/nixos/modules/services/matrix/draupnir.nix" + # "${nixpkgs-Draupnir}/nixos/modules/services/matrix/draupnir.nix" ]; -# nixpkgs.overlays = [ -# (final: prev: { -# draupnir = nixpkgs-DraupnirPkg.legacyPackages.${pkgs.stdenv.hostPlatform.system}.draupnir; -# }) -# ]; + # nixpkgs.overlays = [ + # (final: prev: { + # draupnir = nixpkgs-DraupnirPkg.legacyPackages.${pkgs.stdenv.hostPlatform.system}.draupnir; + # }) + # ]; }; hostAddress = "192.168.100.1"; localAddress = "192.168.100.19"; diff --git a/host/Rory-ovh/services/containers/draupnir-cme/container.nix b/host/Rory-ovh/services/containers/draupnir-cme/container.nix index b1ee74b..4b2d0da 100755 --- a/host/Rory-ovh/services/containers/draupnir-cme/container.nix +++ b/host/Rory-ovh/services/containers/draupnir-cme/container.nix @@ -1,4 +1,4 @@ -{ +{ #nixpkgs-Draupnir, #nixpkgs-DraupnirPkg, ... @@ -20,11 +20,11 @@ ./services/draupnir.nix #"${nixpkgs-Draupnir}/nixos/modules/services/matrix/draupnir.nix" ]; -# nixpkgs.overlays = [ -# (final: prev: { -# draupnir = nixpkgs-DraupnirPkg.legacyPackages.${pkgs.stdenv.hostPlatform.system}.draupnir; -# }) -# ]; + # nixpkgs.overlays = [ + # (final: prev: { + # draupnir = nixpkgs-DraupnirPkg.legacyPackages.${pkgs.stdenv.hostPlatform.system}.draupnir; + # }) + # ]; }; hostAddress = "192.168.100.1"; localAddress = "192.168.100.17"; diff --git a/host/Rory-ovh/services/containers/draupnir-fedora/container.nix b/host/Rory-ovh/services/containers/draupnir-fedora/container.nix index 5ba9765..fe4571e 100644 --- a/host/Rory-ovh/services/containers/draupnir-fedora/container.nix +++ b/host/Rory-ovh/services/containers/draupnir-fedora/container.nix @@ -1,6 +1,6 @@ { -# nixpkgs-Draupnir, -# nixpkgs-DraupnirPkg, + # nixpkgs-Draupnir, + # nixpkgs-DraupnirPkg, ... }: @@ -8,8 +8,8 @@ privateNetwork = true; autoStart = true; specialArgs = { -# inherit nixpkgs-Draupnir; -# inherit nixpkgs-DraupnirPkg; + # inherit nixpkgs-Draupnir; + # inherit nixpkgs-DraupnirPkg; }; config = { lib, pkgs, ... }: @@ -18,13 +18,13 @@ ../shared.nix ./root.nix ./services/draupnir.nix -# "${nixpkgs-Draupnir}/nixos/modules/services/matrix/draupnir.nix" + # "${nixpkgs-Draupnir}/nixos/modules/services/matrix/draupnir.nix" ]; -# nixpkgs.overlays = [ -# (final: prev: { -# draupnir = nixpkgs-DraupnirPkg.legacyPackages.${pkgs.stdenv.hostPlatform.system}.draupnir; -# }) -# ]; + # nixpkgs.overlays = [ + # (final: prev: { + # draupnir = nixpkgs-DraupnirPkg.legacyPackages.${pkgs.stdenv.hostPlatform.system}.draupnir; + # }) + # ]; }; hostAddress = "192.168.100.1"; localAddress = "192.168.100.18"; diff --git a/host/Rory-ovh/services/containers/syntest1/container.nix b/host/Rory-ovh/services/containers/syntest1/container.nix new file mode 100644 index 0000000..dc84478 --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest1/container.nix @@ -0,0 +1,73 @@ +{ + nixpkgs-master, + matrix-synapse-unwrapped-patched, + ... +}: + +{ + privateNetwork = true; + autoStart = true; + specialArgs = { + inherit matrix-synapse-unwrapped-patched; + }; + nixpkgs = nixpkgs-master; + config = + { + lib, + pkgs, + matrix-synapse-unwrapped-patched, + ... + }: + { + imports = [ + ../shared.nix + ./root.nix + ./services/synapse/synapse-main.nix + ./services/postgres.nix + ( + { + pkgs, + lib, + config, + ... + }: + + { + options.virtualisation = { + isVmVariant = lib.mkOption { + default = false; + example = true; + description = "Whether this build is a VM build."; + type = lib.types.bool; + }; + }; + config = { + virtualisation.vmVariant = { + virtualisation.isVmVariant = true; + }; + }; + } + ) + ]; + + nixpkgs.overlays = [ + (final: prev: { + matrix-synapse-unwrapped = matrix-synapse-unwrapped-patched; + }) + ]; + }; + hostAddress = "192.168.100.1"; + localAddress = "192.168.100.20"; + + bindMounts."postgres" = { + hostPath = "/data/dedicated/postgres-syntest1"; + mountPoint = "/data/postgres"; + isReadOnly = false; + }; + + #bindMounts."synapse-shared-secret" = { + # hostPath = "/data/secrets/synapse-shared-secret"; + # mountPoint = "/data/secrets/synapse-shared-secret"; + # isReadOnly = false; + #}; +} diff --git a/host/Rory-ovh/services/containers/syntest1/root.nix b/host/Rory-ovh/services/containers/syntest1/root.nix new file mode 100644 index 0000000..0ebce9e --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest1/root.nix @@ -0,0 +1,16 @@ +{ ... }: + +{ + networking.useHostResolvConf = true; + + networking.hosts = { + "192.168.100.1" = [ + "matrix.rory.gay" + "rory.gay" + ]; + }; + + networking.firewall = { + enable = true; + }; +} diff --git a/host/Rory-ovh/services/containers/syntest1/services/postgres.nix b/host/Rory-ovh/services/containers/syntest1/services/postgres.nix new file mode 100755 index 0000000..98d7155 --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest1/services/postgres.nix @@ -0,0 +1,99 @@ +{ config, pkgs, ... }: + +{ + systemd.tmpfiles.rules = [ "d /data/postgres 0750 postgres postgres" ]; + + services.postgresql = { + enable = true; + package = pkgs.postgresql_17_jit; + enableTCPIP = true; + authentication = pkgs.lib.mkOverride 10 '' + # TYPE, DATABASE, USER, ADDRESS, METHOD + local all all trust + host all all 127.0.0.1/32 trust + host all all ::1/128 trust + host discordbots discordbots 192.168.1.2/32 trust + host matrix-synapse-rory-gay matrix-synapse-rory-gay 192.168.1.5/32 trust + host all all 0.0.0.0/0 md5 + ''; + # initialScript = pkgs.writeText "backend-initScript" '' + # CREATE ROLE nixcloud WITH LOGIN PASSWORD 'nixcloud' CREATEDB; + # CREATE DATABASE nixcloud; + # GRANT ALL PRIVILEGES ON DATABASE nixcloud TO nixcloud; + # ''; + dataDir = "/data/postgres"; + settings = { + # https://pgconfigurator.cybertec.at/ + max_connections = 2500; + superuser_reserved_connections = 3; + + shared_buffers = if config.virtualisation.isVmVariant then "128MB" else "64GB"; + work_mem = if config.virtualisation.isVmVariant then "64MB" else "32GB"; + maintenance_work_mem = if config.virtualisation.isVmVariant then "512MB" else "8GB"; + huge_pages = "try"; + effective_cache_size = if config.virtualisation.isVmVariant then "1GB" else "64GB"; # was 22 + effective_io_concurrency = 100; + random_page_cost = 1.1; + + # can use this to view stats: SELECT query, total_time, calls, rows FROM pg_stat_statements ORDER BY total_time DESC LIMIT 10; + shared_preload_libraries = "pg_stat_statements"; + track_io_timing = "on"; + track_functions = "pl"; + "pg_stat_statements.max" = "10000"; # additional + "pg_stat_statements.track" = "all"; # additional + + wal_level = "replica"; + max_wal_senders = 0; + synchronous_commit = "on"; # was ond3 + + checkpoint_timeout = "15min"; + checkpoint_completion_target = "0.9"; + max_wal_size = "2GB"; + min_wal_size = "1GB"; + + wal_compression = "off"; + wal_buffers = "-1"; + wal_writer_delay = "500ms"; # was 100 + wal_writer_flush_after = "32MB"; # was 1 + #checkpoint_segments = "64"; # additional + default_statistics_target = "250"; # additional + + bgwriter_delay = "200ms"; + bgwriter_lru_maxpages = "100"; + bgwriter_lru_multiplier = "2.0"; + bgwriter_flush_after = "0"; + + max_worker_processes = "64"; # was 14 + max_parallel_workers_per_gather = "32"; # was 7 + max_parallel_maintenance_workers = "32"; # was 7 + max_parallel_workers = "64"; # was 14 + parallel_leader_participation = "on"; + + enable_partitionwise_join = "on"; + enable_partitionwise_aggregate = "on"; + jit = "on"; + max_slot_wal_keep_size = "1GB"; + track_wal_io_timing = "on"; + maintenance_io_concurrency = "4"; + wal_recycle = "on"; + + }; + }; + + # services.prometheus.exporters.postgres = { + # enable = true; + # port = 9187; + # extraFlags = [ + # "--collector.database_wraparound" + # "--collector.long_running_transactions" + # "--collector.postmaster" + # "--collector.process_idle" + # "--collector.stat_activity_autovacuum" + # "--collector.stat_statements" + # #"--collector.stat_wal_receiver" #we dont have WAL receivers + # "--collector.statio_user_indexes" + # "--collector.xlog_location" + # ]; + # }; + +} diff --git a/host/Rory-ovh/services/containers/syntest1/services/synapse/caches.nix b/host/Rory-ovh/services/containers/syntest1/services/synapse/caches.nix new file mode 100644 index 0000000..7ddb804 --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest1/services/synapse/caches.nix @@ -0,0 +1,25 @@ +{ + gc_min_interval = [ + "15m" + "30m" + "60m" + ]; + gc_thresholds = [ + 10000 + + 5000 + 2500 + ]; + event_cache_size = "12000K"; # defaults to 10K + caches = { + global_factor = 500000.0; + cache_entry_ttl = "24h"; + expire_caches = true; + sync_response_cache_duration = "15m"; + cache_autotuning = { + max_cache_memory_usage = "65536M"; + target_cache_memory_usage = "32768M"; + min_cache_ttl = "6h"; + }; + }; +} diff --git a/host/Rory-ovh/services/containers/syntest1/services/synapse/db.nix b/host/Rory-ovh/services/containers/syntest1/services/synapse/db.nix new file mode 100644 index 0000000..28dc79f --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest1/services/synapse/db.nix @@ -0,0 +1,49 @@ +{ + workerName ? null, + dbGroup ? null, +}: +{ + name = "psycopg2"; + args = { + user = "matrix-synapse"; + password = "somepassword"; + database = "matrix-synapse"; + host = "/run/postgresql"; + application_name = "matrix-synapse - ${if workerName == null then throw "synapse/db.nix: workerName unspecified" else workerName}"; + cp_min = + if dbGroup == "solo" then + 1 + else if dbGroup == "small" then + 2 + else if dbGroup == "medium" then + 5 + else if dbGroup == "large" then + 10 + else + throw "synapse/db.nix: Invalid dbGroup: ${if dbGroup == null then "null" else dbGroup}"; + cp_max = + if dbGroup == "solo" then + 1 + else if dbGroup == "small" then + 2 + else if dbGroup == "medium" then + 10 + else if dbGroup == "large" then + 10 + else + throw "synapse/db.nix: Invalid dbGroup: ${if dbGroup == null then "null" else dbGroup}"; + + # cp_reconnect - default=True - https://github.com/element-hq/synapse/blob/develop/synapse/storage/database.py#L129 + # cp_noisy - default=False - https://docs.twisted.org/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__ - info logs during operation + # check_same_thread - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/config/database.py#L65 - can this even be set? + }; + + # synchronous_commit - default=True - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L56 + # statement_timeout - default=60 * 60 * 1000 ms - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L63 + # allow_unsafe_locale - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L99 + # allow_outdated_version - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L92 - needs source link + # txn_limit - default=0 - https://github.com/element-hq/synapse/blob/develop/synapse/storage/database.py#L564 + + statement_timeout = 24 * 60 * 60 * 1000; # 24 hours, good for bg jobs + txn_limit = 500; # maybe dropping old data from pg caches helps? +} diff --git a/host/Rory-ovh/services/containers/syntest1/services/synapse/ratelimits.nix b/host/Rory-ovh/services/containers/syntest1/services/synapse/ratelimits.nix new file mode 100644 index 0000000..85f51f2 --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest1/services/synapse/ratelimits.nix @@ -0,0 +1,97 @@ +{ + # messages + rc_message = { + per_second = 1000000; + burst_count = 1000000; + }; + rc_admin_redaction = { + per_second = 10000000; + burst_count = 10000000; + }; + + # room joins + rc_joins = { + local = { + per_second = 1000; + burst_count = 1000; + }; + remote = { + per_second = 1000; + burst_count = 1000; + }; + }; + rc_joins_per_room = { + per_second = 1000; + burst_count = 1000; + }; + + # room invites + rc_invites = { + per_room = { + per_second = 1000; + burst_count = 1000; + }; + per_user = { + per_second = 1000; + burst_count = 1000; + }; + per_issuer = { + per_second = 1000; + burst_count = 1000; + }; + }; + rc_third_party_invite = { + per_second = 1000; + burst_count = 1000; + }; + + # federation + rc_federation = { + window_size = 10; + sleep_limit = 1000; + sleep_delay = 100; + reject_limit = 1000; + concurrent = 100; + }; + federation_rr_transactions_per_room_per_second = 100; + + # media + rc_media_create = { + per_second = 1000; + burst_count = 1000; + }; + remote_media_download_burst_count = "512G"; + remote_media_download_per_second = "512G"; + + # authentication + rc_login = { + address = { + per_second = 1000; + burst_count = 1000; + }; + account = { + per_second = 1000; + burst_count = 1000; + }; + failed_attempts = { + per_second = 0.1; + burst_count = 3; + }; + }; + rc_3pid_validation = { + per_second = 1000; + burst_count = 1000; + }; + + #presence + rc_presence.per_user = { + per_second = 1; + burst_count = 2; + }; + + #delayed events + rc_delayed_event_mgmt = { + per_second = 1000; + burst_count = 1000; + }; +} diff --git a/host/Rory-ovh/services/containers/syntest1/services/synapse/synapse-main.nix b/host/Rory-ovh/services/containers/syntest1/services/synapse/synapse-main.nix new file mode 100755 index 0000000..66648e6 --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest1/services/synapse/synapse-main.nix @@ -0,0 +1,212 @@ +{ + config, + pkgs, + matrix-synapse-patched, + ... +}: + +{ + services.matrix-synapse = { + enable = true; + withJemalloc = true; + dataDir = "/data/matrix-synapse"; + + settings = + { + server_name = "syntest1.rory.gay"; + + dummy_events_threshold = 5; + cleanup_extremities_with_dummy_events = true; + + enable_registration = true; + registration_requires_token = true; + + require_membership_for_aliases = false; + redaction_retention_period = null; + user_ips_max_age = null; + allow_device_name_lookup_over_federation = true; + + federation = { + client_timeout = "90s"; # 30 # default=60s + max_short_retries = 6; # 12 + max_short_retry_delay = "10s"; # 5 + max_long_retries = 5; + max_long_retry_delay = "30s"; + + # rapid retry, small increments + destination_min_retry_interval = "1m"; # default=10m + destination_max_retry_interval = "12h"; # default=7d + destination_retry_multiplier = 1.1; # 1.2 # default=2 + }; + + registration_shared_secret_path = "/data/secrets/synapse-shared-secret"; + + listeners = [ + { + port = 8008; + bind_addresses = [ "127.0.0.1" ]; + type = "http"; + tls = false; + x_forwarded = true; + resources = [ + { + names = [ + "client" + "federation" + ]; + compress = false; + } + ]; + } + { + type = "http"; + path = "/run/matrix-synapse/main.sock"; + resources = [ + { + names = [ "replication" ]; + compress = false; + } + ]; + } + ]; + presence = { + enable = true; + update_interval = 60; + }; + database = ( + import ./db.nix { + workerName = "main"; + dbGroup = "medium"; + } + ); + + #region Media + max_upload_size = "512M"; + + max_avatar_size = "512M"; + max_image_pixels = "250M"; + + max_pending_media_uploads = 512; + dynamic_thumbnails = true; + + prevent_media_downloads_from = [ + # none, give me all the media + ]; + enable_authenticated_media = false; + + url_preview_enabled = true; + max_spider_size = "50M"; + + #endregion + + ui_auth = { + session_timeout = "1m"; + }; + + login_via_existing_session = { + enabled = true; + require_ui_auth = true; + token_timeout = "1y"; + }; + + report_stats = false; + + user_directory = { + enabled = true; + search_all_users = true; + prefer_local_users = true; + }; + + # https://github.com/element-hq/synapse/blob/master/synapse/config/experimental.py + experimental_features = { + "msc2409_to_device_messages_enabled" = true; + "msc2815_enabled" = true; # Redacted event content + "msc3026_enabled" = true; # Busy presence + "msc3202_transaction_extensions" = true; # appservice transaction extensions (device list/keys) + "msc3266_enabled" = true; # Room summary API + "msc3391_enabled" = true; # Remove account data + "msc3823_account_suspension" = true; # Account suspension + "msc3852_enabled" = true; # Last seen on /devices (CS-API/admin) + "msc3874_enabled" = true; # filtering /messages with rel_types / not_rel_types + "msc3890_enabled" = true; # communicate account data deletion to clients + "msc3912_enabled" = true; # /messages with rel_types / not_rel_types and event id + "msc3916_authenticated_media_enabled" = true; # Authenticated media + "msc4069_profile_inhibit_propagation" = true; # Inhibit profile update propagation + "msc4133_enabled" = true; # Custom profile fields + "msc4151_enabled" = true; # Report room API (CS-API) + "msc4210_enabled" = false; # Remove legacy mentions -- we want this *disabled* for moderation reasons + "msc4222_enabled" = true; # state_after in sync + }; + + redis = { + enabled = true; + path = "/run/redis-matrix-synapse/redis.sock"; + }; + + instance_map = { + main = { + # replication listener + path = "/run/matrix-synapse/main.sock"; + }; + }; + } + // import ./ratelimits.nix + // import ./caches.nix; + }; + + services.redis = { + package = pkgs.valkey; + servers.matrix-synapse = { + enable = true; + user = "matrix-synapse"; + }; + }; + + systemd.tmpfiles.rules = [ + "D /run/redis-matrix-synapse 0755 matrix-synapse matrix-synapse" + "D /run/secrets 0755 nobody nobody" + ]; + + systemd.services."matrix-synapse-generate-token" = { + # generate /data/secrets/synapse-shared-secret + description = "Generate Synapse shared secret"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + before = [ "matrix-synapse.service" ]; + script = '' + set -e -x -o pipefail + echo "Starting key generation" + if [ ! -f "/data/secrets/synapse-shared-secret" ] + then + echo "Generating new key" + ${pkgs.openssl}/bin/openssl rand -base64 32 > /data/secrets/synapse-shared-secret + echo "Key generation complete" + else + echo "Not generating key, key exists" + fi + echo "Script complete" + ''; + }; + + systemd.services."matrix-synapse-postgres-init" = { + description = "Generate synapse postgres user"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network.target" + "postgresql.service" + ]; + before = [ "matrix-synapse.service" ]; + + script = '' + set -e -x -o pipefail + ${pkgs.postgresql}/bin/createuser ${config.services.matrix-synapse.settings.database.args.user} || true + ${pkgs.postgresql}/bin/createdb --encoding=UTF8 --locale=C --template=template0 --owner=${config.services.matrix-synapse.settings.database.args.user} ${config.services.matrix-synapse.settings.database.args.database} || true + ''; + serviceConfig = { + User = "postgres"; + Group = "postgres"; + WorkingDirectory = config.services.postgresql.dataDir; + RemainAfterExit = true; + }; + }; +} diff --git a/host/Rory-ovh/services/containers/syntest2/container.nix b/host/Rory-ovh/services/containers/syntest2/container.nix new file mode 100644 index 0000000..0e657bc --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest2/container.nix @@ -0,0 +1,63 @@ +{ + nixpkgs-master, + ... +}: + +{ + privateNetwork = true; + autoStart = true; + specialArgs = { }; + nixpkgs = nixpkgs-master; + config = + { + lib, + pkgs, + ... + }: + { + imports = [ + ../shared.nix + ./root.nix + ./services/synapse/synapse-main.nix + ./services/postgres.nix + ( + { + pkgs, + lib, + config, + ... + }: + + { + options.virtualisation = { + isVmVariant = lib.mkOption { + default = false; + example = true; + description = "Whether this build is a VM build."; + type = lib.types.bool; + }; + }; + config = { + virtualisation.vmVariant = { + virtualisation.isVmVariant = true; + }; + }; + } + ) + ]; + }; + hostAddress = "192.168.100.1"; + localAddress = "192.168.100.21"; + + bindMounts."postgres" = { + hostPath = "/data/dedicated/postgres-syntest2"; + mountPoint = "/data/postgres"; + isReadOnly = false; + }; + + #bindMounts."synapse-shared-secret" = { + # hostPath = "/data/secrets/synapse-shared-secret"; + # mountPoint = "/data/secrets/synapse-shared-secret"; + # isReadOnly = false; + #}; +} diff --git a/host/Rory-ovh/services/containers/syntest2/root.nix b/host/Rory-ovh/services/containers/syntest2/root.nix new file mode 100644 index 0000000..0ebce9e --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest2/root.nix @@ -0,0 +1,16 @@ +{ ... }: + +{ + networking.useHostResolvConf = true; + + networking.hosts = { + "192.168.100.1" = [ + "matrix.rory.gay" + "rory.gay" + ]; + }; + + networking.firewall = { + enable = true; + }; +} diff --git a/host/Rory-ovh/services/containers/syntest2/services/postgres.nix b/host/Rory-ovh/services/containers/syntest2/services/postgres.nix new file mode 100755 index 0000000..98d7155 --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest2/services/postgres.nix @@ -0,0 +1,99 @@ +{ config, pkgs, ... }: + +{ + systemd.tmpfiles.rules = [ "d /data/postgres 0750 postgres postgres" ]; + + services.postgresql = { + enable = true; + package = pkgs.postgresql_17_jit; + enableTCPIP = true; + authentication = pkgs.lib.mkOverride 10 '' + # TYPE, DATABASE, USER, ADDRESS, METHOD + local all all trust + host all all 127.0.0.1/32 trust + host all all ::1/128 trust + host discordbots discordbots 192.168.1.2/32 trust + host matrix-synapse-rory-gay matrix-synapse-rory-gay 192.168.1.5/32 trust + host all all 0.0.0.0/0 md5 + ''; + # initialScript = pkgs.writeText "backend-initScript" '' + # CREATE ROLE nixcloud WITH LOGIN PASSWORD 'nixcloud' CREATEDB; + # CREATE DATABASE nixcloud; + # GRANT ALL PRIVILEGES ON DATABASE nixcloud TO nixcloud; + # ''; + dataDir = "/data/postgres"; + settings = { + # https://pgconfigurator.cybertec.at/ + max_connections = 2500; + superuser_reserved_connections = 3; + + shared_buffers = if config.virtualisation.isVmVariant then "128MB" else "64GB"; + work_mem = if config.virtualisation.isVmVariant then "64MB" else "32GB"; + maintenance_work_mem = if config.virtualisation.isVmVariant then "512MB" else "8GB"; + huge_pages = "try"; + effective_cache_size = if config.virtualisation.isVmVariant then "1GB" else "64GB"; # was 22 + effective_io_concurrency = 100; + random_page_cost = 1.1; + + # can use this to view stats: SELECT query, total_time, calls, rows FROM pg_stat_statements ORDER BY total_time DESC LIMIT 10; + shared_preload_libraries = "pg_stat_statements"; + track_io_timing = "on"; + track_functions = "pl"; + "pg_stat_statements.max" = "10000"; # additional + "pg_stat_statements.track" = "all"; # additional + + wal_level = "replica"; + max_wal_senders = 0; + synchronous_commit = "on"; # was ond3 + + checkpoint_timeout = "15min"; + checkpoint_completion_target = "0.9"; + max_wal_size = "2GB"; + min_wal_size = "1GB"; + + wal_compression = "off"; + wal_buffers = "-1"; + wal_writer_delay = "500ms"; # was 100 + wal_writer_flush_after = "32MB"; # was 1 + #checkpoint_segments = "64"; # additional + default_statistics_target = "250"; # additional + + bgwriter_delay = "200ms"; + bgwriter_lru_maxpages = "100"; + bgwriter_lru_multiplier = "2.0"; + bgwriter_flush_after = "0"; + + max_worker_processes = "64"; # was 14 + max_parallel_workers_per_gather = "32"; # was 7 + max_parallel_maintenance_workers = "32"; # was 7 + max_parallel_workers = "64"; # was 14 + parallel_leader_participation = "on"; + + enable_partitionwise_join = "on"; + enable_partitionwise_aggregate = "on"; + jit = "on"; + max_slot_wal_keep_size = "1GB"; + track_wal_io_timing = "on"; + maintenance_io_concurrency = "4"; + wal_recycle = "on"; + + }; + }; + + # services.prometheus.exporters.postgres = { + # enable = true; + # port = 9187; + # extraFlags = [ + # "--collector.database_wraparound" + # "--collector.long_running_transactions" + # "--collector.postmaster" + # "--collector.process_idle" + # "--collector.stat_activity_autovacuum" + # "--collector.stat_statements" + # #"--collector.stat_wal_receiver" #we dont have WAL receivers + # "--collector.statio_user_indexes" + # "--collector.xlog_location" + # ]; + # }; + +} diff --git a/host/Rory-ovh/services/containers/syntest2/services/synapse/caches.nix b/host/Rory-ovh/services/containers/syntest2/services/synapse/caches.nix new file mode 100644 index 0000000..7ddb804 --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest2/services/synapse/caches.nix @@ -0,0 +1,25 @@ +{ + gc_min_interval = [ + "15m" + "30m" + "60m" + ]; + gc_thresholds = [ + 10000 + + 5000 + 2500 + ]; + event_cache_size = "12000K"; # defaults to 10K + caches = { + global_factor = 500000.0; + cache_entry_ttl = "24h"; + expire_caches = true; + sync_response_cache_duration = "15m"; + cache_autotuning = { + max_cache_memory_usage = "65536M"; + target_cache_memory_usage = "32768M"; + min_cache_ttl = "6h"; + }; + }; +} diff --git a/host/Rory-ovh/services/containers/syntest2/services/synapse/db.nix b/host/Rory-ovh/services/containers/syntest2/services/synapse/db.nix new file mode 100644 index 0000000..28dc79f --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest2/services/synapse/db.nix @@ -0,0 +1,49 @@ +{ + workerName ? null, + dbGroup ? null, +}: +{ + name = "psycopg2"; + args = { + user = "matrix-synapse"; + password = "somepassword"; + database = "matrix-synapse"; + host = "/run/postgresql"; + application_name = "matrix-synapse - ${if workerName == null then throw "synapse/db.nix: workerName unspecified" else workerName}"; + cp_min = + if dbGroup == "solo" then + 1 + else if dbGroup == "small" then + 2 + else if dbGroup == "medium" then + 5 + else if dbGroup == "large" then + 10 + else + throw "synapse/db.nix: Invalid dbGroup: ${if dbGroup == null then "null" else dbGroup}"; + cp_max = + if dbGroup == "solo" then + 1 + else if dbGroup == "small" then + 2 + else if dbGroup == "medium" then + 10 + else if dbGroup == "large" then + 10 + else + throw "synapse/db.nix: Invalid dbGroup: ${if dbGroup == null then "null" else dbGroup}"; + + # cp_reconnect - default=True - https://github.com/element-hq/synapse/blob/develop/synapse/storage/database.py#L129 + # cp_noisy - default=False - https://docs.twisted.org/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__ - info logs during operation + # check_same_thread - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/config/database.py#L65 - can this even be set? + }; + + # synchronous_commit - default=True - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L56 + # statement_timeout - default=60 * 60 * 1000 ms - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L63 + # allow_unsafe_locale - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L99 + # allow_outdated_version - default=False - https://github.com/element-hq/synapse/blob/develop/synapse/storage/engines/postgres.py#L92 - needs source link + # txn_limit - default=0 - https://github.com/element-hq/synapse/blob/develop/synapse/storage/database.py#L564 + + statement_timeout = 24 * 60 * 60 * 1000; # 24 hours, good for bg jobs + txn_limit = 500; # maybe dropping old data from pg caches helps? +} diff --git a/host/Rory-ovh/services/containers/syntest2/services/synapse/ratelimits.nix b/host/Rory-ovh/services/containers/syntest2/services/synapse/ratelimits.nix new file mode 100644 index 0000000..85f51f2 --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest2/services/synapse/ratelimits.nix @@ -0,0 +1,97 @@ +{ + # messages + rc_message = { + per_second = 1000000; + burst_count = 1000000; + }; + rc_admin_redaction = { + per_second = 10000000; + burst_count = 10000000; + }; + + # room joins + rc_joins = { + local = { + per_second = 1000; + burst_count = 1000; + }; + remote = { + per_second = 1000; + burst_count = 1000; + }; + }; + rc_joins_per_room = { + per_second = 1000; + burst_count = 1000; + }; + + # room invites + rc_invites = { + per_room = { + per_second = 1000; + burst_count = 1000; + }; + per_user = { + per_second = 1000; + burst_count = 1000; + }; + per_issuer = { + per_second = 1000; + burst_count = 1000; + }; + }; + rc_third_party_invite = { + per_second = 1000; + burst_count = 1000; + }; + + # federation + rc_federation = { + window_size = 10; + sleep_limit = 1000; + sleep_delay = 100; + reject_limit = 1000; + concurrent = 100; + }; + federation_rr_transactions_per_room_per_second = 100; + + # media + rc_media_create = { + per_second = 1000; + burst_count = 1000; + }; + remote_media_download_burst_count = "512G"; + remote_media_download_per_second = "512G"; + + # authentication + rc_login = { + address = { + per_second = 1000; + burst_count = 1000; + }; + account = { + per_second = 1000; + burst_count = 1000; + }; + failed_attempts = { + per_second = 0.1; + burst_count = 3; + }; + }; + rc_3pid_validation = { + per_second = 1000; + burst_count = 1000; + }; + + #presence + rc_presence.per_user = { + per_second = 1; + burst_count = 2; + }; + + #delayed events + rc_delayed_event_mgmt = { + per_second = 1000; + burst_count = 1000; + }; +} diff --git a/host/Rory-ovh/services/containers/syntest2/services/synapse/synapse-main.nix b/host/Rory-ovh/services/containers/syntest2/services/synapse/synapse-main.nix new file mode 100755 index 0000000..7d5498f --- /dev/null +++ b/host/Rory-ovh/services/containers/syntest2/services/synapse/synapse-main.nix @@ -0,0 +1,211 @@ +{ + config, + pkgs, + ... +}: + +{ + services.matrix-synapse = { + enable = true; + withJemalloc = true; + dataDir = "/data/matrix-synapse"; + + settings = + { + server_name = "syntest2.rory.gay"; + + dummy_events_threshold = 5; + cleanup_extremities_with_dummy_events = true; + + enable_registration = true; + registration_requires_token = true; + + require_membership_for_aliases = false; + redaction_retention_period = null; + user_ips_max_age = null; + allow_device_name_lookup_over_federation = true; + + federation = { + client_timeout = "90s"; # 30 # default=60s + max_short_retries = 6; # 12 + max_short_retry_delay = "10s"; # 5 + max_long_retries = 5; + max_long_retry_delay = "30s"; + + # rapid retry, small increments + destination_min_retry_interval = "1m"; # default=10m + destination_max_retry_interval = "12h"; # default=7d + destination_retry_multiplier = 1.1; # 1.2 # default=2 + }; + + registration_shared_secret_path = "/data/secrets/synapse-shared-secret"; + + listeners = [ + { + port = 8008; + bind_addresses = [ "127.0.0.1" ]; + type = "http"; + tls = false; + x_forwarded = true; + resources = [ + { + names = [ + "client" + "federation" + ]; + compress = false; + } + ]; + } + { + type = "http"; + path = "/run/matrix-synapse/main.sock"; + resources = [ + { + names = [ "replication" ]; + compress = false; + } + ]; + } + ]; + presence = { + enable = true; + update_interval = 60; + }; + database = ( + import ./db.nix { + workerName = "main"; + dbGroup = "medium"; + } + ); + + #region Media + max_upload_size = "512M"; + + max_avatar_size = "512M"; + max_image_pixels = "250M"; + + max_pending_media_uploads = 512; + dynamic_thumbnails = true; + + prevent_media_downloads_from = [ + # none, give me all the media + ]; + enable_authenticated_media = false; + + url_preview_enabled = true; + max_spider_size = "50M"; + + #endregion + + ui_auth = { + session_timeout = "1m"; + }; + + login_via_existing_session = { + enabled = true; + require_ui_auth = true; + token_timeout = "1y"; + }; + + report_stats = false; + + user_directory = { + enabled = true; + search_all_users = true; + prefer_local_users = true; + }; + + # https://github.com/element-hq/synapse/blob/master/synapse/config/experimental.py + experimental_features = { + "msc2409_to_device_messages_enabled" = true; + "msc2815_enabled" = true; # Redacted event content + "msc3026_enabled" = true; # Busy presence + "msc3202_transaction_extensions" = true; # appservice transaction extensions (device list/keys) + "msc3266_enabled" = true; # Room summary API + "msc3391_enabled" = true; # Remove account data + "msc3823_account_suspension" = true; # Account suspension + "msc3852_enabled" = true; # Last seen on /devices (CS-API/admin) + "msc3874_enabled" = true; # filtering /messages with rel_types / not_rel_types + "msc3890_enabled" = true; # communicate account data deletion to clients + "msc3912_enabled" = true; # /messages with rel_types / not_rel_types and event id + "msc3916_authenticated_media_enabled" = true; # Authenticated media + "msc4069_profile_inhibit_propagation" = true; # Inhibit profile update propagation + "msc4133_enabled" = true; # Custom profile fields + "msc4151_enabled" = true; # Report room API (CS-API) + "msc4210_enabled" = false; # Remove legacy mentions -- we want this *disabled* for moderation reasons + "msc4222_enabled" = true; # state_after in sync + }; + + redis = { + enabled = true; + path = "/run/redis-matrix-synapse/redis.sock"; + }; + + instance_map = { + main = { + # replication listener + path = "/run/matrix-synapse/main.sock"; + }; + }; + } + // import ./ratelimits.nix + // import ./caches.nix; + }; + + services.redis = { + package = pkgs.valkey; + servers.matrix-synapse = { + enable = true; + user = "matrix-synapse"; + }; + }; + + systemd.tmpfiles.rules = [ + "D /run/redis-matrix-synapse 0755 matrix-synapse matrix-synapse" + "D /run/secrets 0755 nobody nobody" + ]; + + systemd.services."matrix-synapse-generate-token" = { + # generate /data/secrets/synapse-shared-secret + description = "Generate Synapse shared secret"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + before = [ "matrix-synapse.service" ]; + script = '' + set -e -x -o pipefail + echo "Starting key generation" + if [ ! -f "/data/secrets/synapse-shared-secret" ] + then + echo "Generating new key" + ${pkgs.openssl}/bin/openssl rand -base64 32 > /data/secrets/synapse-shared-secret + echo "Key generation complete" + else + echo "Not generating key, key exists" + fi + echo "Script complete" + ''; + }; + + systemd.services."matrix-synapse-postgres-init" = { + description = "Generate synapse postgres user"; + wantedBy = [ "multi-user.target" ]; + after = [ + "network.target" + "postgresql.service" + ]; + before = [ "matrix-synapse.service" ]; + + script = '' + set -e -x -o pipefail + ${pkgs.postgresql}/bin/createuser ${config.services.matrix-synapse.settings.database.args.user} || true + ${pkgs.postgresql}/bin/createdb --encoding=UTF8 --locale=C --template=template0 --owner=${config.services.matrix-synapse.settings.database.args.user} ${config.services.matrix-synapse.settings.database.args.database} || true + ''; + serviceConfig = { + User = "postgres"; + Group = "postgres"; + WorkingDirectory = config.services.postgresql.dataDir; + RemainAfterExit = true; + }; + }; +} diff --git a/host/Rory-ovh/services/email/maddy.nix b/host/Rory-ovh/services/email/maddy.nix index 548cb1a..3f1d427 100644 --- a/host/Rory-ovh/services/email/maddy.nix +++ b/host/Rory-ovh/services/email/maddy.nix @@ -35,8 +35,8 @@ loader = "file"; certificates = [ { - # certPath = "/var/lib/acme/mail.rory.gay/fullchain.pem"; - # keyPath = "/var/lib/acme/mail.rory.gay/key.pem"; + # certPath = "/var/lib/acme/mail.rory.gay/fullchain.pem"; + # keyPath = "/var/lib/acme/mail.rory.gay/key.pem"; certPath = "/run/credentials/maddy.service/acme-fullchain.pem"; keyPath = "/run/credentials/maddy.service/acme-key.pem"; } @@ -60,7 +60,7 @@ fsType = "none"; options = [ "bind" ]; }; - + systemd.services.maddy.serviceConfig = { LoadCredential = [ "acme-fullchain.pem:/var/lib/acme/rory.gay/fullchain.pem" diff --git a/host/Rory-ovh/services/mastodon.nix b/host/Rory-ovh/services/mastodon.nix index 56f1808..5febe3b 100644 --- a/host/Rory-ovh/services/mastodon.nix +++ b/host/Rory-ovh/services/mastodon.nix @@ -5,8 +5,8 @@ enable = true; webProcesses = 8; webThreads = 4; - - streamingProcesses = 63; - localDomain = "rory.gay"; + + streamingProcesses = 63; + localDomain = "rory.gay"; }; } diff --git a/host/Rory-ovh/services/matrix/synapse/caches.nix b/host/Rory-ovh/services/matrix/synapse/caches.nix index f00c78c..7ddb804 100644 --- a/host/Rory-ovh/services/matrix/synapse/caches.nix +++ b/host/Rory-ovh/services/matrix/synapse/caches.nix @@ -6,7 +6,7 @@ ]; gc_thresholds = [ 10000 - + 5000 2500 ]; diff --git a/host/Rory-ovh/services/matrix/synapse/synapse-main.nix b/host/Rory-ovh/services/matrix/synapse/synapse-main.nix index 7907927..65dba4e 100755 --- a/host/Rory-ovh/services/matrix/synapse/synapse-main.nix +++ b/host/Rory-ovh/services/matrix/synapse/synapse-main.nix @@ -207,7 +207,7 @@ # before = [ "matrix-synapse.service" ]; # So the registration can be used by Synapse # wantedBy = [ "multi-user.target" ]; # after = [ "network.target" ]; -# + # # script = '' # set -e -x -o pipefail # echo "Starting key generation" @@ -264,7 +264,10 @@ systemd.services."matrix-synapse-postgres-init" = { description = "Generate synapse postgres user"; wantedBy = [ "multi-user.target" ]; - after = [ "network.target" "postgresql.service" ]; + after = [ + "network.target" + "postgresql.service" + ]; before = [ "matrix-synapse.service" ]; script = '' diff --git a/host/Rory-ovh/services/nginx/nginx.nix b/host/Rory-ovh/services/nginx/nginx.nix index fee1573..72fbef0 100755 --- a/host/Rory-ovh/services/nginx/nginx.nix +++ b/host/Rory-ovh/services/nginx/nginx.nix @@ -41,7 +41,7 @@ in #"boorunav.com" = serveDir { path = "/data/nginx/html_boorunav"; }; # "catgirlsaresexy.com" = serveDir { path = "/data/nginx/html_catgirlsaresexy"; }; # "sugarcanemc.org" = serveDir { path = "/data/nginx/html_sugarcanemc"; }; -# + # #"siliconheaven.thearcanebrony.net" = serveDir { path = "/data/nginx/html_siliconheaven"; }; #"lfs.thearcanebrony.net" = serveDir { path = "/data/nginx/html_lfs"; }; #"git.thearcanebrony.net" = serveDir { path = "/data/nginx/html_git"; }; @@ -49,27 +49,29 @@ in #"spigotav.thearcanebrony.net" = serveDir { path = "/data/nginx/html_spigotav"; }; #"terra.thearcanebrony.net" = serveDir { path = "/data/nginx/html_terrarchive"; }; #"vives.thearcanebrony.net" = serveDir { path = "/data/nginx/html_vives"; }; -# + # # "git.rory.gay" = serveDir { path = "/data/nginx/html_git"; }; # "wad.rory.gay" = serveDir { path = "/data/nginx/html_wad"; } // { # locations."/".extraConfig = "autoindex on; try_files $uri $uri/ /index.html;"; # }; # "wad-api.rory.gay" = import ./rory.gay/wad-api.nix; -# + # #"thearcanebrony.net" = import ./thearcanebrony.net/root.nix; # "sentry.thearcanebrony.net" = import ./thearcanebrony.net/sentry.nix; # "search.thearcanebrony.net" = import ./thearcanebrony.net/search.nix; -# + # "rory.gay" = import ./rory.gay/root.nix { inherit config; }; # "lfs.rory.gay" = serveDir { path = "/data/nginx/html_lfs"; }; -# + # # "awooradio.thearcanebrony.net" = import ./thearcanebrony.net/awooradio.nix; "cgit.rory.gay" = import ./rory.gay/cgit.nix { inherit config; }; # #"jitsi.rory.gay" = import ./rory.gay/jitsi.nix; -# + # # #matrix... # "conduit.rory.gay" = import ./rory.gay/conduit.nix; "matrix.rory.gay" = import ./rory.gay/matrix.nix { inherit config; }; + "syntest1.rory.gay" = import ./rory.gay/syntest1.nix { inherit config; }; + "syntest2.rory.gay" = import ./rory.gay/syntest2.nix { inherit config; }; "libmatrix-fed-test.rory.gay" = import ./rory.gay/libmatrix-fed-test.nix { inherit config; }; "safensound.rory.gay" = import ./rory.gay/safensound.nix { inherit config; }; "demo.safensound.rory.gay" = import ./rory.gay/demo.safensound.nix { inherit config; }; @@ -90,6 +92,9 @@ in security.acme.defaults.email = "root@rory.gay"; networking.hosts."127.0.0.1" = builtins.attrNames config.services.nginx.virtualHosts; - networking.firewall.allowedTCPPorts = [ 80 443 ]; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; networking.firewall.allowedUDPPorts = [ 443 ]; } diff --git a/host/Rory-ovh/services/nginx/rory.gay/api.safensound.nix b/host/Rory-ovh/services/nginx/rory.gay/api.safensound.nix index b0ff075..0a44039 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/api.safensound.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/api.safensound.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; diff --git a/host/Rory-ovh/services/nginx/rory.gay/cgit.nix b/host/Rory-ovh/services/nginx/rory.gay/cgit.nix index 7b49a42..7788e64 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/cgit.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/cgit.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; diff --git a/host/Rory-ovh/services/nginx/rory.gay/conduit.matrixunittests.nix b/host/Rory-ovh/services/nginx/rory.gay/conduit.matrixunittests.nix index 231d5e3..b436336 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/conduit.matrixunittests.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/conduit.matrixunittests.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; diff --git a/host/Rory-ovh/services/nginx/rory.gay/demo.safensound.nix b/host/Rory-ovh/services/nginx/rory.gay/demo.safensound.nix index f75c78b..40ecfb0 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/demo.safensound.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/demo.safensound.nix @@ -1,13 +1,13 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; -# quic = true; + # quic = true; http3 = !config.virtualisation.isVmVariant; http3_hq = !config.virtualisation.isVmVariant; kTLS = !config.virtualisation.isVmVariant; root = "/data/nginx/html_safensound_demo"; -# reuseport = true; + # reuseport = true; extraConfig = '' brotli off; brotli_static off; diff --git a/host/Rory-ovh/services/nginx/rory.gay/ec.nix b/host/Rory-ovh/services/nginx/rory.gay/ec.nix index c50b1f9..18cf3e1 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/ec.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/ec.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; diff --git a/host/Rory-ovh/services/nginx/rory.gay/libmatrix-fed-test.nix b/host/Rory-ovh/services/nginx/rory.gay/libmatrix-fed-test.nix index c2909d6..9284e8a 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/libmatrix-fed-test.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/libmatrix-fed-test.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; diff --git a/host/Rory-ovh/services/nginx/rory.gay/matrix-bak.nix b/host/Rory-ovh/services/nginx/rory.gay/matrix-bak.nix index 1af3669..694a521 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/matrix-bak.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/matrix-bak.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; diff --git a/host/Rory-ovh/services/nginx/rory.gay/matrix.nix b/host/Rory-ovh/services/nginx/rory.gay/matrix.nix index 45a507f..609b8af 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/matrix.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/matrix.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; @@ -65,8 +65,4 @@ } }'; ''; - - locations."~ ^/(client/|_matrix/client/unstable/org.matrix.msc3575/sync)" = { - proxyPass = "http://localhost:8100"; - }; } diff --git a/host/Rory-ovh/services/nginx/rory.gay/matrixunittests.nix b/host/Rory-ovh/services/nginx/rory.gay/matrixunittests.nix index f23f0dd..eae32f7 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/matrixunittests.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/matrixunittests.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; diff --git a/host/Rory-ovh/services/nginx/rory.gay/mru.nix b/host/Rory-ovh/services/nginx/rory.gay/mru.nix index 6e685de..cb010b4 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/mru.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/mru.nix @@ -1,13 +1,13 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; -# quic = true; + # quic = true; http3 = !config.virtualisation.isVmVariant; http3_hq = !config.virtualisation.isVmVariant; kTLS = !config.virtualisation.isVmVariant; root = "/data/nginx/html_mru"; -# reuseport = true; + # reuseport = true; extraConfig = '' brotli off; brotli_static off; diff --git a/host/Rory-ovh/services/nginx/rory.gay/root.nix b/host/Rory-ovh/services/nginx/rory.gay/root.nix index a7720ec..2f491cf 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/root.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/root.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; diff --git a/host/Rory-ovh/services/nginx/rory.gay/safensound.nix b/host/Rory-ovh/services/nginx/rory.gay/safensound.nix index 9208129..5c3cfd7 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/safensound.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/safensound.nix @@ -1,13 +1,13 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; -# quic = true; + # quic = true; http3 = !config.virtualisation.isVmVariant; http3_hq = !config.virtualisation.isVmVariant; kTLS = !config.virtualisation.isVmVariant; root = "/data/nginx/html_safensound"; -# reuseport = true; + # reuseport = true; extraConfig = '' brotli off; brotli_static off; diff --git a/host/Rory-ovh/services/nginx/rory.gay/stream.nix b/host/Rory-ovh/services/nginx/rory.gay/stream.nix index caed22f..cad3d13 100755 --- a/host/Rory-ovh/services/nginx/rory.gay/stream.nix +++ b/host/Rory-ovh/services/nginx/rory.gay/stream.nix @@ -1,4 +1,4 @@ -{ config }: +{ config }: { enableACME = !config.virtualisation.isVmVariant; addSSL = !config.virtualisation.isVmVariant; @@ -8,26 +8,26 @@ proxyWebsockets = true; recommendedProxySettings = true; extraConfig = '' - proxy_ssl_verify off; -# proxy_set_header Host youthapp.inuits.dev; - proxy_ssl_server_name on; + proxy_ssl_verify off; + # proxy_set_header Host youthapp.inuits.dev; + proxy_ssl_server_name on; - more_set_headers 'Access-Control-Allow-Origin: *'; - more_set_headers 'Access-Control-Allow-Methods: *'; - # - # Custom headers and headers various browsers *should* be OK with but aren't - # - more_set_headers 'Access-Control-Allow-Headers: *, Authorization'; - # - # Tell client that this pre-flight info is valid for 20 days - # - more_set_headers 'Access-Control-Max-Age: 1728000'; + more_set_headers 'Access-Control-Allow-Origin: *'; + more_set_headers 'Access-Control-Allow-Methods: *'; + # + # Custom headers and headers various browsers *should* be OK with but aren't + # + more_set_headers 'Access-Control-Allow-Headers: *, Authorization'; + # + # Tell client that this pre-flight info is valid for 20 days + # + more_set_headers 'Access-Control-Max-Age: 1728000'; - if ($request_method = 'OPTIONS') { - more_set_headers 'Content-Type: text/plain; charset=utf-8'; - more_set_headers 'Content-Length: 0'; - return 204; - } + if ($request_method = 'OPTIONS') { + more_set_headers 'Content-Type: text/plain; charset=utf-8'; + more_set_headers 'Content-Length: 0'; + return 204; + } ''; }; }; diff --git a/host/Rory-ovh/services/nginx/rory.gay/syntest1.nix b/host/Rory-ovh/services/nginx/rory.gay/syntest1.nix new file mode 100755 index 0000000..7c60eb0 --- /dev/null +++ b/host/Rory-ovh/services/nginx/rory.gay/syntest1.nix @@ -0,0 +1,68 @@ +{ config }: +{ + enableACME = !config.virtualisation.isVmVariant; + addSSL = !config.virtualisation.isVmVariant; + locations."/" = { + #proxyPass = "http://127.0.0.1:9001"; + proxyPass = "http://192.168.100.20:8008"; + extraConfig = '' + if ($request_method = 'OPTIONS') { + more_set_headers 'Access-Control-Allow-Origin: *'; + more_set_headers 'Access-Control-Allow-Methods: *'; + # + # Custom headers and headers various browsers *should* be OK with but aren't + # + more_set_headers 'Access-Control-Allow-Headers: *, Authorization'; + # + # Tell client that this pre-flight info is valid for 20 days + # + more_set_headers 'Access-Control-Max-Age: 1728000'; + more_set_headers 'Content-Type: text/plain; charset=utf-8'; + more_set_headers 'Content-Length: 0'; + return 204; + } + ''; + }; + + locations."= /.well-known/matrix/server".extraConfig = '' + more_set_headers 'Content-Type application/json'; + more_set_headers 'Access-Control-Allow-Origin *'; + return 200 '${builtins.toJSON { "m.server" = "syntest1.rory.gay:443"; }}'; + ''; + locations."= /.well-known/matrix/client".extraConfig = '' + more_set_headers 'Content-Type application/json'; + more_set_headers 'Access-Control-Allow-Origin *'; + return 200 '${ + builtins.toJSON { + "m.homeserver".base_url = "https://syntest1.rory.gay"; + "org.matrix.msc3575.proxy".url = "https://syntest1.rory.gay"; + } + }'; + ''; + locations."= /.well-known/matrix/support".extraConfig = '' + more_set_headers 'Content-Type application/json'; + more_set_headers 'Access-Control-Allow-Origin *'; + return 200 '${ + builtins.toJSON { + admins = [ + { + matrix_id = "@emma:rory.gay"; + role = "admin"; + } + { + matrix_id = "@alicia:rory.gay"; + role = "admin"; + } + { + matrix_id = "@root:rory.gay"; + role = "admin"; + } + { + matrix_id = "@rory:rory.gay"; + role = "admin"; + } + ]; + } + }'; + ''; +} diff --git a/host/Rory-ovh/services/nginx/rory.gay/syntest2.nix b/host/Rory-ovh/services/nginx/rory.gay/syntest2.nix new file mode 100755 index 0000000..f1dbcc4 --- /dev/null +++ b/host/Rory-ovh/services/nginx/rory.gay/syntest2.nix @@ -0,0 +1,68 @@ +{ config }: +{ + enableACME = !config.virtualisation.isVmVariant; + addSSL = !config.virtualisation.isVmVariant; + locations."/" = { + #proxyPass = "http://127.0.0.1:9001"; + proxyPass = "http://192.168.100.21:8008"; + extraConfig = '' + if ($request_method = 'OPTIONS') { + more_set_headers 'Access-Control-Allow-Origin: *'; + more_set_headers 'Access-Control-Allow-Methods: *'; + # + # Custom headers and headers various browsers *should* be OK with but aren't + # + more_set_headers 'Access-Control-Allow-Headers: *, Authorization'; + # + # Tell client that this pre-flight info is valid for 20 days + # + more_set_headers 'Access-Control-Max-Age: 1728000'; + more_set_headers 'Content-Type: text/plain; charset=utf-8'; + more_set_headers 'Content-Length: 0'; + return 204; + } + ''; + }; + + locations."= /.well-known/matrix/server".extraConfig = '' + more_set_headers 'Content-Type application/json'; + more_set_headers 'Access-Control-Allow-Origin *'; + return 200 '${builtins.toJSON { "m.server" = "syntest2.rory.gay:443"; }}'; + ''; + locations."= /.well-known/matrix/client".extraConfig = '' + more_set_headers 'Content-Type application/json'; + more_set_headers 'Access-Control-Allow-Origin *'; + return 200 '${ + builtins.toJSON { + "m.homeserver".base_url = "https://syntest2.rory.gay"; + "org.matrix.msc3575.proxy".url = "https://syntest2.rory.gay"; + } + }'; + ''; + locations."= /.well-known/matrix/support".extraConfig = '' + more_set_headers 'Content-Type application/json'; + more_set_headers 'Access-Control-Allow-Origin *'; + return 200 '${ + builtins.toJSON { + admins = [ + { + matrix_id = "@emma:rory.gay"; + role = "admin"; + } + { + matrix_id = "@alicia:rory.gay"; + role = "admin"; + } + { + matrix_id = "@root:rory.gay"; + role = "admin"; + } + { + matrix_id = "@rory:rory.gay"; + role = "admin"; + } + ]; + } + }'; + ''; +} diff --git a/host/RoryNix/configuration.nix b/host/RoryNix/configuration.nix index 891d3ad..8e3c0cc 100644 --- a/host/RoryNix/configuration.nix +++ b/host/RoryNix/configuration.nix @@ -147,7 +147,10 @@ zsh-completions ]; - fonts.packages = with pkgs; [ nerd-fonts.jetbrains-mono noto-fonts-monochrome-emoji ]; + fonts.packages = with pkgs; [ + nerd-fonts.jetbrains-mono + noto-fonts-monochrome-emoji + ]; nix = { settings = { experimental-features = [ diff --git a/host/uISO/development.nix b/host/uISO/development.nix index 90ffcfb..6148c76 100644 --- a/host/uISO/development.nix +++ b/host/uISO/development.nix @@ -2,7 +2,7 @@ config, lib, pkgs, - + #params #enableBios ? true, #enableEfi ? true, @@ -25,7 +25,7 @@ #splashImage = null; grubTheme = null; }; - + boot = { initrd = { #systemd.enable = true; @@ -48,4 +48,4 @@ btop neofetch ]; -} \ No newline at end of file +} diff --git a/host/uISO/iso-root.nix b/host/uISO/iso-root.nix index 6f02772..5c1ff5d 100644 --- a/host/uISO/iso-root.nix +++ b/host/uISO/iso-root.nix @@ -3,7 +3,7 @@ lib, pkgs, nixpkgs, - + #params #enableBios ? true, #enableEfi ? true, @@ -19,7 +19,7 @@ (nixpkgs + "/nixos/modules/installer/cd-dvd/iso-image.nix") ]; - fileSystems = lib.mkImageMediaOverride config.lib.isoFileSystems;# // { + fileSystems = lib.mkImageMediaOverride config.lib.isoFileSystems; # // { # "/".device = lib.mkForce "/dev/disk/by-label/NIXOS_ISO"; # "/nix/.ro-store".device = lib.mkForce "/sysroot/iso/nix-store.squashfs"; #}; @@ -31,16 +31,15 @@ makeUsbBootable = false; makeBiosBootable = true; }; - - boot.supportedFilesystems = lib.mkForce [ ]; + boot.supportedFilesystems = lib.mkForce [ ]; hardware.enableRedistributableFirmware = lib.mkForce false; #environment.systemPackages = lib.mkForce [ ]; documentation.enable = lib.mkForce false; documentation.nixos.enable = lib.mkForce false; networking.wireless.enable = lib.mkForce false; - system.extraDependencies = lib.mkForce []; + system.extraDependencies = lib.mkForce [ ]; boot = { loader = { @@ -56,7 +55,6 @@ enableContainers = lib.mkForce false; }; - #perlless profile system.switch.enable = lib.mkForce false; @@ -86,12 +84,12 @@ networking.networkmanager.enable = false; systemd.coredump.enable = false; services.timesyncd.enable = false; - + services.nscd.enableNsncd = false; networking.dhcpcd.enable = false; services.udev.enable = false; services.nscd.enable = false; - system.nssModules = lib.mkForce []; + system.nssModules = lib.mkForce [ ]; systemd.oomd.enable = false; #services.getty = { @@ -119,10 +117,10 @@ boot.initrd.systemd.suppressedUnits = [ "systemd-hibernate-clear.service" ]; - + systemd.suppressedSystemUnits = [ "systemd-hibernate-clear.service" "systemd-bootctl@.service" "systemd-bootctl.socket" ]; -} \ No newline at end of file +} diff --git a/host/uISO/overlays/grub-overlay.nix b/host/uISO/overlays/grub-overlay.nix index abbed8e..14a36e8 100644 --- a/host/uISO/overlays/grub-overlay.nix +++ b/host/uISO/overlays/grub-overlay.nix @@ -1,18 +1,22 @@ final: prev: { - grub2 = (prev.grub2.override { - zfsSupport = false; - efiSupport = false; - xenSupport = false; - }).overrideAttrs (oldAttrs: { - doCheck = false; - doInstallCheck = false; - #remove --enable-grub-mount - configureFlags = oldAttrs.configureFlags ++ [ "--disable-year2038 --disable-nls --disable-rpath --disable-dependency-tracking --disable-grub-mount --disable-grub-themes --disable-grub-mkfont" ]; - # remove unnecessary commands - # postInstall = oldAttrs.postInstall + '' - # rm -rf $out/share/locale - # ''; - }); + grub2 = + (prev.grub2.override { + zfsSupport = false; + efiSupport = false; + xenSupport = false; + }).overrideAttrs + (oldAttrs: { + doCheck = false; + doInstallCheck = false; + #remove --enable-grub-mount + configureFlags = oldAttrs.configureFlags ++ [ + "--disable-year2038 --disable-nls --disable-rpath --disable-dependency-tracking --disable-grub-mount --disable-grub-themes --disable-grub-mkfont" + ]; + # remove unnecessary commands + # postInstall = oldAttrs.postInstall + '' + # rm -rf $out/share/locale + # ''; + }); grub2_light = final.grub2; grub2_efi = final.grub2; diff --git a/host/uISO/overlays/systemd-overlay.nix b/host/uISO/overlays/systemd-overlay.nix index 69a4d3f..a1ddadd 100644 --- a/host/uISO/overlays/systemd-overlay.nix +++ b/host/uISO/overlays/systemd-overlay.nix @@ -18,7 +18,7 @@ final: prev: { withFido2 = false; withHomed = false; withHostnamed = false; - withHwdb = true; # required by nixos + withHwdb = true; # required by nixos withImportd = false; withIptables = false; withKmod = true; # required by nixos diff --git a/host/uISO/pkgs/resize.nix b/host/uISO/pkgs/resize.nix index 5dc3b77..8ffd598 100644 --- a/host/uISO/pkgs/resize.nix +++ b/host/uISO/pkgs/resize.nix @@ -4,6 +4,9 @@ derivation { name = "resize"; version = "1.0"; builder = "${pkgs.bash}/bin/bash"; - args = [ "-c" "${pkgs.coreutils}/bin/mkdir -p $out/bin; ${pkgs.coreutils}/bin/cp ${pkgs.xterm}/bin/.resize-wrapped $out/bin/resize" ]; + args = [ + "-c" + "${pkgs.coreutils}/bin/mkdir -p $out/bin; ${pkgs.coreutils}/bin/cp ${pkgs.xterm}/bin/.resize-wrapped $out/bin/resize" + ]; system = builtins.currentSystem; -} \ No newline at end of file +} diff --git a/modules/base.nix b/modules/base.nix index 44909aa..386ba73 100755 --- a/modules/base.nix +++ b/modules/base.nix @@ -175,10 +175,10 @@ virtualisation = { memorySize = 8192; cores = 6; - msize = 1*1024*1024; + msize = 1 * 1024 * 1024; bios = pkgs.qboot; }; - + monitoring.monitorAll = lib.mkForce false; services.promtail.enable = lib.mkForce false; networking.useDHCP = lib.mkOverride 51 true; diff --git a/modules/software-templates/steam.nix b/modules/software-templates/steam.nix index ac548a7..2054b7d 100644 --- a/modules/software-templates/steam.nix +++ b/modules/software-templates/steam.nix @@ -1,4 +1,5 @@ -{ pkgs, ... }: { +{ pkgs, ... }: +{ programs.steam = { enable = true; gamescopeSession.enable = true; @@ -7,9 +8,9 @@ proton-ge-bin ]; }; - + environment.systemPackages = with pkgs; [ steam-run steam-acf ]; -} \ No newline at end of file +} diff --git a/modules/software-templates/synapse-workers/generic.nix b/modules/software-templates/synapse-workers/generic.nix index 7fa3967..899f156 100644 --- a/modules/software-templates/synapse-workers/generic.nix +++ b/modules/software-templates/synapse-workers/generic.nix @@ -29,8 +29,8 @@ let ++ lib.optional (lib.elem "stream_events" tasks) "events"; # recursive update list of attrs -# recursiveMerge = list: lib.foldl (a: b: lib.recursiveUpdate a b) (lib.head list) (lib.tail list); -# workerRoutes = recursiveMerge (lib.map (type: workerLib.workerRoutes.${type}) streamTypes); + # recursiveMerge = list: lib.foldl (a: b: lib.recursiveUpdate a b) (lib.head list) (lib.tail list); + # workerRoutes = recursiveMerge (lib.map (type: workerLib.workerRoutes.${type}) streamTypes); in { config = lib.mkIf (cfg.accountDataStreamWriters > 0) { @@ -84,7 +84,9 @@ in ]; }) enabledResources; database = lib.recursiveUpdate (lib.recursiveUpdate config.services.matrix-synapse.settings.database { - application_name = "matrix-synapse (${config.services.matrix-synapse.settings.server_name}) - ${if workerName == null then throw "synapse/db.nix: workerName unspecified" else workerName}"; + application_name = "matrix-synapse (${config.services.matrix-synapse.settings.server_name}) - ${ + if workerName == null then throw "synapse/db.nix: workerName unspecified" else workerName + }"; }) dbOverrides; #region Media diff --git a/modules/software-templates/synapse-workers/generic/auth.nix b/modules/software-templates/synapse-workers/generic/auth.nix index 47064be..2ba5901 100644 --- a/modules/software-templates/synapse-workers/generic/auth.nix +++ b/modules/software-templates/synapse-workers/generic/auth.nix @@ -23,7 +23,7 @@ in }; }) workers ); - + media_instance_running_background_jobs = lib.mkIf (lib.elem "media_repo_background" tasks) "${workerName}"; enable_media_repo = lib.mkIf (lib.elem "media_repo" tasks) false; }; diff --git a/modules/software-templates/synapse-workers/generic/client-reader.nix b/modules/software-templates/synapse-workers/generic/client-reader.nix index 9072988..3192860 100644 --- a/modules/software-templates/synapse-workers/generic/client-reader.nix +++ b/modules/software-templates/synapse-workers/generic/client-reader.nix @@ -23,7 +23,7 @@ in }; }) workers ); - + media_instance_running_background_jobs = lib.mkIf (lib.elem "media_repo_background" tasks) "${workerName}"; enable_media_repo = lib.mkIf (lib.elem "media_repo" tasks) false; }; diff --git a/modules/software-templates/synapse-workers/generic/event-creator.nix b/modules/software-templates/synapse-workers/generic/event-creator.nix index e035405..6f1f4b3 100644 --- a/modules/software-templates/synapse-workers/generic/event-creator.nix +++ b/modules/software-templates/synapse-workers/generic/event-creator.nix @@ -23,7 +23,7 @@ in }; }) workers ); - + media_instance_running_background_jobs = lib.mkIf (lib.elem "media_repo_background" tasks) "${workerName}"; enable_media_repo = lib.mkIf (lib.elem "media_repo" tasks) false; }; diff --git a/modules/software-templates/synapse-workers/generic/federation-inbound.nix b/modules/software-templates/synapse-workers/generic/federation-inbound.nix index af18e8e..8fa991d 100644 --- a/modules/software-templates/synapse-workers/generic/federation-inbound.nix +++ b/modules/software-templates/synapse-workers/generic/federation-inbound.nix @@ -23,7 +23,7 @@ in }; }) workers ); - + media_instance_running_background_jobs = lib.mkIf (lib.elem "media_repo_background" tasks) "${workerName}"; enable_media_repo = lib.mkIf (lib.elem "media_repo" tasks) false; }; diff --git a/modules/software-templates/synapse-workers/generic/federation-reader.nix b/modules/software-templates/synapse-workers/generic/federation-reader.nix index 09e8419..acedf32 100644 --- a/modules/software-templates/synapse-workers/generic/federation-reader.nix +++ b/modules/software-templates/synapse-workers/generic/federation-reader.nix @@ -23,7 +23,7 @@ in }; }) workers ); - + media_instance_running_background_jobs = lib.mkIf (lib.elem "media_repo_background" tasks) "${workerName}"; enable_media_repo = lib.mkIf (lib.elem "media_repo" tasks) false; }; diff --git a/modules/software-templates/synapse-workers/generic/federation-sender.nix b/modules/software-templates/synapse-workers/generic/federation-sender.nix index bf6cf51..ce4e75d 100644 --- a/modules/software-templates/synapse-workers/generic/federation-sender.nix +++ b/modules/software-templates/synapse-workers/generic/federation-sender.nix @@ -5,7 +5,7 @@ let workers = lib.range 0 (cfg.federationSenders - 1); workerName = "federation_sender"; tasks = [ ]; - workerRoutes = {}; + workerRoutes = { }; enabledResources = lib.attrNames workerRoutes; in { @@ -21,16 +21,16 @@ in }; }) workers ); - + # Federation sender send_federation = lib.mkIf (lib.elem "federation_sender" tasks) false; federation_sender_instances = lib.mkIf (lib.elem "federation_sender" tasks) [ workerName ]; outbound_federation_restricted_to = lib.mkIf (lib.elem "federation_sender" tasks) [ workerName ]; worker_replication_secret = "${workerName}_secret"; - + # Pusher pusher_instances = lib.optional (lib.elem "pusher" tasks) "${workerName}"; - + # Media repo media_instance_running_background_jobs = lib.mkIf (lib.elem "media_repo_background" tasks) "${workerName}"; enable_media_repo = lib.mkIf (lib.elem "media_repo" tasks) false; @@ -71,7 +71,7 @@ in workerName = "${workerName}-${toString index}"; } ); - + # Media repo enable_media_repo = lib.elem "media_repo" tasks; rc_federation = { diff --git a/modules/software-templates/synapse-workers/generic/pusher.nix b/modules/software-templates/synapse-workers/generic/pusher.nix index 8c6b697..5d5b4ff 100644 --- a/modules/software-templates/synapse-workers/generic/pusher.nix +++ b/modules/software-templates/synapse-workers/generic/pusher.nix @@ -5,7 +5,7 @@ let workers = lib.range 0 (cfg.pushers - 1); workerName = "pusher"; tasks = [ ]; - workerRoutes = {}; + workerRoutes = { }; enabledResources = lib.attrNames workerRoutes; in { diff --git a/modules/software-templates/synapse-workers/generic/sync.nix b/modules/software-templates/synapse-workers/generic/sync.nix index eb47e59..cafe4e2 100644 --- a/modules/software-templates/synapse-workers/generic/sync.nix +++ b/modules/software-templates/synapse-workers/generic/sync.nix @@ -23,7 +23,7 @@ in }; }) workers ); - + media_instance_running_background_jobs = lib.mkIf (lib.elem "media_repo_background" tasks) "${workerName}"; enable_media_repo = lib.mkIf (lib.elem "media_repo" tasks) false; }; diff --git a/modules/software-templates/synapse-workers/lib.nix b/modules/software-templates/synapse-workers/lib.nix index c99e2ef..010ff8a 100644 --- a/modules/software-templates/synapse-workers/lib.nix +++ b/modules/software-templates/synapse-workers/lib.nix @@ -129,16 +129,16 @@ presence.client = [ "~ ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/" ]; - + accountData.client = [ "~ ^/_matrix/client/(r0|v3|unstable)/account_data" "~ ^/_matrix/client/(r0|v3|unstable)/rooms/.*/account_data" "~ ^/_matrix/client/(r0|v3|unstable)/rooms/.*/tags" ]; - + userDirectory.client = [ "~ ^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$" - + #profile "~ ^/_matrix/client/v3/profile/.*$" "~ ^/_matrix/client/v3/profile/.*/(displayname|avatar_url)$" diff --git a/modules/software-templates/synapse-workers/single/appservice.nix b/modules/software-templates/synapse-workers/single/appservice.nix index 03a080b..871a7c1 100644 --- a/modules/software-templates/synapse-workers/single/appservice.nix +++ b/modules/software-templates/synapse-workers/single/appservice.nix @@ -6,7 +6,7 @@ let dbGroup = "small"; workerName = "appservice"; tasks = [ "appservice" ]; - workerRoutes = {}; + workerRoutes = { }; enabledResources = lib.attrNames workerRoutes; in { diff --git a/modules/software-templates/synapse-workers/stream-writers/account_data-stream-writer.nix b/modules/software-templates/synapse-workers/stream-writers/account_data-stream-writer.nix index 3f8363a..33ccc25 100644 --- a/modules/software-templates/synapse-workers/stream-writers/account_data-stream-writer.nix +++ b/modules/software-templates/synapse-workers/stream-writers/account_data-stream-writer.nix @@ -4,7 +4,7 @@ let cfg = config.services.matrix-synapse; workerLib = import ../lib.nix; dbGroup = "medium"; -# streamWriterType = "account_data"; + # streamWriterType = "account_data"; workers = lib.range 0 (cfg.accountDataStreamWriters - 1); workerName = "account_data_stream_writer"; tasks = [ "stream_account_data" ]; @@ -12,7 +12,8 @@ let in let enabledResources = lib.attrNames workerRoutes; - streamTypes = [] + streamTypes = + [ ] ++ lib.optional (lib.elem "stream_account_data" tasks) "account_data" ++ lib.optional (lib.elem "stream_presence" tasks) "presence" ++ lib.optional (lib.elem "stream_push_rules" tasks) "push_rules" @@ -36,7 +37,7 @@ in ); #stream_writers.${streamWriterType} = lib.map (index: "${workerName}-${toString index}") workers; -# stream_writers = lib.listToA + # stream_writers = lib.listToA # map `streams` to `workers` stream_writers = lib.listToAttrs ( lib.map (stream: { diff --git a/modules/software-templates/synapse-workers/stream-writers/event-stream-writer.nix b/modules/software-templates/synapse-workers/stream-writers/event-stream-writer.nix index 6abce3f..3c72d46 100644 --- a/modules/software-templates/synapse-workers/stream-writers/event-stream-writer.nix +++ b/modules/software-templates/synapse-workers/stream-writers/event-stream-writer.nix @@ -7,8 +7,8 @@ let workers = lib.range 0 (cfg.eventStreamWriters - 1); workerName = "event_stream_writer"; tasks = [ ]; - workerRoutes = {}; - enabledResources = []; + workerRoutes = { }; + enabledResources = [ ]; in { config = lib.mkIf (cfg.eventStreamWriters > 0) { diff --git a/modules/users/Rory.nix b/modules/users/Rory.nix index 9667a3b..dbb2c4a 100755 --- a/modules/users/Rory.nix +++ b/modules/users/Rory.nix @@ -133,7 +133,7 @@ alias ytdl='nix run ${pkgs.path}\#yt-dlp -- -k --console-title --print-traffic' alias ytdl-mp3='nix run ${pkgs.path}\#yt-dlp -- -k --console-title --print-traffic -x --audio-quality 0 --audio-format mp3 -i' alias ytdl-mp3l='nix run ${pkgs.path}\#yt-dlp -- --console-title -x --audio-quality 0 --audio-format mp3 -o "%%(playlist)s/%%(playlist_index)s. %%(title)s.%%(ext)s" --download-archive archive.txt --embed-thumbnail --add-metadata -R 100 --http-chunk-size 64M --buffer-size 64M -i' - + [ -f "$HOME/.profile" ] && . $HOME/.profile ''; #alias knconfig='cp .config ../$(date ''+%Y%m%d_%k%M%S\'').config -v; make CC=clang LLVM=1 nconfig' diff --git a/packages/matrix-spec.nix b/packages/matrix-spec.nix index 01475bf..17cc4a3 100644 --- a/packages/matrix-spec.nix +++ b/packages/matrix-spec.nix @@ -5,10 +5,10 @@ buildGoModule, hugo, cacert, - + pkgs, - - matrixSpecSrc + + matrixSpecSrc, }: let src = matrixSpecSrc; @@ -27,12 +27,12 @@ buildGoModule { # Nix doesn't play well with Hugo's "GitInfo" module, so disable it and inject # the revision from the flake. postPatch = '' -# substituteInPlace ./site/layouts/shortcodes/gitinfo.html \ -# --replace "{{ .Page.GitInfo.Hash }}" "${rev}" + # substituteInPlace ./site/layouts/shortcodes/gitinfo.html \ + # --replace "{{ .Page.GitInfo.Hash }}" "${rev}" -# substituteInPlace ./config/_default/config.yaml \ -# --replace "enableGitInfo: true" "enableGitInfo: false" - ${lib.getExe pkgs.lsd} -lAh . + # substituteInPlace ./config/_default/config.yaml \ + # --replace "enableGitInfo: true" "enableGitInfo: false" + ${lib.getExe pkgs.lsd} -lAh . ''; # Generate the Hugo site before building the Go application which embeds the diff --git a/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch b/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch new file mode 100644 index 0000000..0d92e4e --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch @@ -0,0 +1,100 @@ +From 1b82f35b613e96c56bf18015e33f34328ad73188 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Tue, 22 Jul 2025 05:07:01 +0200 +Subject: [PATCH 01/11] Fast auth links + +--- + synapse/storage/database.py | 43 +++++++++++++++++++ + .../databases/main/event_federation.py | 8 ++-- + 2 files changed, 47 insertions(+), 4 deletions(-) + +diff --git a/synapse/storage/database.py b/synapse/storage/database.py +index 6188195614..79af139543 100644 +--- a/synapse/storage/database.py ++++ b/synapse/storage/database.py +@@ -2558,6 +2558,49 @@ class DatabasePool: + + return txn.fetchall() + ++# requires database_engine.supports_using_any_list to be true ++def make_select_id_if_found_sql_clause( ++ database_engine: BaseDatabaseEngine, ++ column: str, ++ table: str, ++ iterable: Collection[Any], ++ *, ++ negative: bool = False, ++) -> Tuple[str, list]: ++ """Returns an SQL clause that checks the given column is in the iterable. ++ ++ On SQLite this expands to `column IN (?, ?, ...)`, whereas on Postgres ++ it expands to `column = ANY(?)`. While both DBs support the `IN` form, ++ using the `ANY` form on postgres means that it views queries with ++ different length iterables as the same, helping the query stats. ++ ++ Args: ++ database_engine ++ column: Name of the column ++ table: Name of the table ++ iterable: The values to check the column against. ++ negative: Whether we should check for inequality, i.e. `NOT IN` ++ ++ Returns: ++ A tuple of SQL query and the args ++ """ ++ # This should hopefully be faster, but also makes postgres query ++ # stats easier to understand. ++ if database_engine.supports_using_any_list: ++ if not negative: ++ clause = f"{column}_lookup AS {column} FROM UNNEST(?::bigint[]) {column}_lookup WHERE EXISTS(SELECT FROM {table} WHERE {column}={column}_lookup)" ++ else: ++ clause = f"{column}_lookup AS {column} FROM UNNEST(?::bigint[]) {column}_lookup WHERE NOT EXISTS(SELECT FROM {table} WHERE {column}={column}_lookup)" ++ ++ return clause, [list(iterable)] ++ else: ++ params = ",".join("?" for _ in iterable) ++ if not negative: ++ clause = f"DISTINCT {column} FROM {table} WHERE {column} IN ({params})" ++ else: ++ clause = f"DISTINCT {column} FROM {table} WHERE {column} NOT IN ({params})" ++ return clause, list(iterable) ++ + + def make_in_list_sql_clause( + database_engine: BaseDatabaseEngine, +diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py +index 46aa5902d8..026f0114ed 100644 +--- a/synapse/storage/databases/main/event_federation.py ++++ b/synapse/storage/databases/main/event_federation.py +@@ -52,6 +52,7 @@ from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, ++ make_select_id_if_found_sql_clause, + ) + from synapse.storage.databases.main.events_worker import EventsWorkerStore + from synapse.storage.databases.main.signatures import SignatureWorkerStore +@@ -362,8 +363,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas + sql = """ + WITH RECURSIVE links(chain_id) AS ( + SELECT +- DISTINCT origin_chain_id +- FROM event_auth_chain_links WHERE %s ++ %s + UNION + SELECT + target_chain_id +@@ -380,8 +380,8 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas + while chains_to_fetch: + batch2 = tuple(itertools.islice(chains_to_fetch, 1000)) + chains_to_fetch.difference_update(batch2) +- clause, args = make_in_list_sql_clause( +- txn.database_engine, "origin_chain_id", batch2 ++ clause, args = make_select_id_if_found_sql_clause( ++ txn.database_engine, "origin_chain_id", "event_auth_chain_links", batch2 + ) + txn.execute(sql % (clause,), args) + +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0001-Hotfix-ignore-rejected-events-in-delayed_events.patch b/packages/overlays/matrix-synapse/patches/0001-Hotfix-ignore-rejected-events-in-delayed_events.patch deleted file mode 100644 index 10bda30..0000000 --- a/packages/overlays/matrix-synapse/patches/0001-Hotfix-ignore-rejected-events-in-delayed_events.patch +++ /dev/null @@ -1,31 +0,0 @@ -From c9685b56adfec0f8917e2a04b792519c57f0baa4 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Sun, 20 Apr 2025 00:30:29 +0200 -Subject: [PATCH 01/10] Hotfix: ignore rejected events in delayed_events - ---- - synapse/handlers/delayed_events.py | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py -index 80cb1cec9b..cb2a34ff73 100644 ---- a/synapse/handlers/delayed_events.py -+++ b/synapse/handlers/delayed_events.py -@@ -208,8 +208,13 @@ class DelayedEventsHandler: - ) - - event = await self._store.get_event( -- delta.event_id, check_room_id=delta.room_id -+ delta.event_id, check_room_id=delta.room_id, allow_rejected=True, allow_none=True - ) -+ -+ if event is None or event.rejected_reason is not None: -+ # This event has been rejected, so we don't want to cancel any delayed events for it. -+ continue -+ - sender = UserID.from_string(event.sender) - - next_send_ts = await self._store.cancel_delayed_state_events( --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0002-Add-too-much-logging-to-room-summary-over-federation.patch b/packages/overlays/matrix-synapse/patches/0002-Add-too-much-logging-to-room-summary-over-federation.patch deleted file mode 100644 index 8441823..0000000 --- a/packages/overlays/matrix-synapse/patches/0002-Add-too-much-logging-to-room-summary-over-federation.patch +++ /dev/null @@ -1,77 +0,0 @@ -From 12f5c44e7d1cedc9f11402fc5c06ce54a8c24915 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Wed, 23 Apr 2025 17:53:52 +0200 -Subject: [PATCH 02/10] Add too much logging to room summary over federation - -Signed-off-by: Rory& ---- - synapse/handlers/room_summary.py | 40 ++++++++++++++++++++++++++++---- - 1 file changed, 36 insertions(+), 4 deletions(-) - -diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py -index 91b131d09b..6e64930682 100644 ---- a/synapse/handlers/room_summary.py -+++ b/synapse/handlers/room_summary.py -@@ -700,23 +700,55 @@ class RoomSummaryHandler: - """ - # The API doesn't return the room version so assume that a - # join rule of knock is valid. -+ join_rule = room.get("join_rule") -+ world_readable = room.get("world_readable") -+ -+ logger.warning( -+ "[EMMA] Checking if room %s is accessible to %s: join_rule=%s, world_readable=%s", -+ room_id, requester, join_rule, world_readable -+ ) -+ - if ( -- room.get("join_rule", JoinRules.PUBLIC) -- in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) -- or room.get("world_readable") is True -+ join_rule in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) -+ or world_readable is True - ): - return True -- elif not requester: -+ else: -+ logger.warning( -+ "[EMMA] Room %s is not accessible to %s: join_rule=%s, world_readable=%s, join_rule result=%s, world_readable result=%s", -+ room_id, requester, join_rule, world_readable, -+ join_rule in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED), -+ world_readable is True -+ ) -+ -+ if not requester: -+ logger.warning( -+ "[EMMA] No requester, so room %s is not accessible", -+ room_id -+ ) - return False -+ - - # Check if the user is a member of any of the allowed rooms from the response. - allowed_rooms = room.get("allowed_room_ids") -+ logger.warning( -+ "[EMMA] Checking if room %s is in allowed rooms for %s: join_rule=%s, allowed_rooms=%s", -+ requester, -+ room_id, -+ join_rule, -+ allowed_rooms -+ ) - if allowed_rooms and isinstance(allowed_rooms, list): - if await self._event_auth_handler.is_user_in_rooms( - allowed_rooms, requester - ): - return True - -+ logger.warning( -+ "[EMMA] Checking if room %s is accessble to %s via local state", -+ room_id, -+ requester -+ ) - # Finally, check locally if we can access the room. The user might - # already be in the room (if it was a child room), or there might be a - # pending invite, etc. --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch b/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch new file mode 100644 index 0000000..3d5ea60 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch @@ -0,0 +1,31 @@ +From 346fb5899fa42d4604b7bf0261c5e1774e6d2c04 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Sun, 20 Apr 2025 00:30:29 +0200 +Subject: [PATCH 02/11] Hotfix: ignore rejected events in delayed_events + +--- + synapse/handlers/delayed_events.py | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py +index 80cb1cec9b..cb2a34ff73 100644 +--- a/synapse/handlers/delayed_events.py ++++ b/synapse/handlers/delayed_events.py +@@ -208,8 +208,13 @@ class DelayedEventsHandler: + ) + + event = await self._store.get_event( +- delta.event_id, check_room_id=delta.room_id ++ delta.event_id, check_room_id=delta.room_id, allow_rejected=True, allow_none=True + ) ++ ++ if event is None or event.rejected_reason is not None: ++ # This event has been rejected, so we don't want to cancel any delayed events for it. ++ continue ++ + sender = UserID.from_string(event.sender) + + next_send_ts = await self._store.cancel_delayed_state_events( +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch b/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch new file mode 100644 index 0000000..5f4e596 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch @@ -0,0 +1,77 @@ +From 929d1e329ec26d2e351591206a82c6e235660437 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Wed, 23 Apr 2025 17:53:52 +0200 +Subject: [PATCH 03/11] Add too much logging to room summary over federation + +Signed-off-by: Rory& +--- + synapse/handlers/room_summary.py | 40 ++++++++++++++++++++++++++++---- + 1 file changed, 36 insertions(+), 4 deletions(-) + +diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py +index 1f322ac263..59b63b2e7c 100644 +--- a/synapse/handlers/room_summary.py ++++ b/synapse/handlers/room_summary.py +@@ -715,23 +715,55 @@ class RoomSummaryHandler: + """ + # The API doesn't return the room version so assume that a + # join rule of knock is valid. ++ join_rule = room.get("join_rule") ++ world_readable = room.get("world_readable") ++ ++ logger.warning( ++ "[EMMA] Checking if room %s is accessible to %s: join_rule=%s, world_readable=%s", ++ room_id, requester, join_rule, world_readable ++ ) ++ + if ( +- room.get("join_rule", JoinRules.PUBLIC) +- in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) +- or room.get("world_readable") is True ++ join_rule in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) ++ or world_readable is True + ): + return True +- elif not requester: ++ else: ++ logger.warning( ++ "[EMMA] Room %s is not accessible to %s: join_rule=%s, world_readable=%s, join_rule result=%s, world_readable result=%s", ++ room_id, requester, join_rule, world_readable, ++ join_rule in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED), ++ world_readable is True ++ ) ++ ++ if not requester: ++ logger.warning( ++ "[EMMA] No requester, so room %s is not accessible", ++ room_id ++ ) + return False ++ + + # Check if the user is a member of any of the allowed rooms from the response. + allowed_rooms = room.get("allowed_room_ids") ++ logger.warning( ++ "[EMMA] Checking if room %s is in allowed rooms for %s: join_rule=%s, allowed_rooms=%s", ++ requester, ++ room_id, ++ join_rule, ++ allowed_rooms ++ ) + if allowed_rooms and isinstance(allowed_rooms, list): + if await self._event_auth_handler.is_user_in_rooms( + allowed_rooms, requester + ): + return True + ++ logger.warning( ++ "[EMMA] Checking if room %s is accessble to %s via local state", ++ room_id, ++ requester ++ ) + # Finally, check locally if we can access the room. The user might + # already be in the room (if it was a child room), or there might be a + # pending invite, etc. +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0003-Log-entire-room-if-accessibility-check-fails.patch b/packages/overlays/matrix-synapse/patches/0003-Log-entire-room-if-accessibility-check-fails.patch deleted file mode 100644 index e1676c6..0000000 --- a/packages/overlays/matrix-synapse/patches/0003-Log-entire-room-if-accessibility-check-fails.patch +++ /dev/null @@ -1,28 +0,0 @@ -From feb88e251b0a7402095643444710f160b9e73daa Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Wed, 23 Apr 2025 18:24:57 +0200 -Subject: [PATCH 03/10] Log entire room if accessibility check fails - -Signed-off-by: Rory& ---- - synapse/handlers/room_summary.py | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py -index 6e64930682..1c39cfed1b 100644 ---- a/synapse/handlers/room_summary.py -+++ b/synapse/handlers/room_summary.py -@@ -916,6 +916,10 @@ class RoomSummaryHandler: - if not room_entry or not await self._is_remote_room_accessible( - requester, room_entry.room_id, room_entry.room - ): -+ logger.warning( -+ "[Emma] Room entry contents: %s", -+ room_entry.room if room_entry else None -+ ) - raise NotFoundError("Room not found or is not accessible") - - room = dict(room_entry.room) --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch b/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch new file mode 100644 index 0000000..290f0da --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch @@ -0,0 +1,28 @@ +From 0ce933278f77e272e2cc894229a1178e1b4fb552 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Wed, 23 Apr 2025 18:24:57 +0200 +Subject: [PATCH 04/11] Log entire room if accessibility check fails + +Signed-off-by: Rory& +--- + synapse/handlers/room_summary.py | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py +index 59b63b2e7c..b65ccdb510 100644 +--- a/synapse/handlers/room_summary.py ++++ b/synapse/handlers/room_summary.py +@@ -931,6 +931,10 @@ class RoomSummaryHandler: + if not room_entry or not await self._is_remote_room_accessible( + requester, room_entry.room_id, room_entry.room + ): ++ logger.warning( ++ "[Emma] Room entry contents: %s", ++ room_entry.room if room_entry else None ++ ) + raise NotFoundError("Room not found or is not accessible") + + room = dict(room_entry.room) +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0004-Log-policy-server-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0004-Log-policy-server-rejected-events.patch deleted file mode 100644 index 63903f1..0000000 --- a/packages/overlays/matrix-synapse/patches/0004-Log-policy-server-rejected-events.patch +++ /dev/null @@ -1,30 +0,0 @@ -From d06fbc3b1b6158a1e3805d3dd282427268dea01a Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Tue, 27 May 2025 05:21:46 +0200 -Subject: [PATCH 04/10] Log policy server rejected events - ---- - synapse/handlers/room_policy.py | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py -index dcfebb128c..3a83c4d6ec 100644 ---- a/synapse/handlers/room_policy.py -+++ b/synapse/handlers/room_policy.py -@@ -84,6 +84,13 @@ class RoomPolicyHandler: - policy_server, event - ) - if recommendation != RECOMMENDATION_OK: -+ logger.info( -+ "[POLICY] Policy server %s recommended not to allow event %s in room %s: %s", -+ policy_server, -+ event.event_id, -+ event.room_id, -+ recommendation, -+ ) - return False - - return True # default allow --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch new file mode 100644 index 0000000..ae59e63 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch @@ -0,0 +1,30 @@ +From 0b5d4c8104bf25f7bbb4e4e7db229742f04199b6 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Tue, 27 May 2025 05:21:46 +0200 +Subject: [PATCH 05/11] Log policy server rejected events + +--- + synapse/handlers/room_policy.py | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py +index 170c477d6f..df5585e4ee 100644 +--- a/synapse/handlers/room_policy.py ++++ b/synapse/handlers/room_policy.py +@@ -87,6 +87,13 @@ class RoomPolicyHandler: + policy_server, event + ) + if recommendation != RECOMMENDATION_OK: ++ logger.info( ++ "[POLICY] Policy server %s recommended not to allow event %s in room %s: %s", ++ policy_server, ++ event.event_id, ++ event.room_id, ++ recommendation, ++ ) + return False + + return True # default allow +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0005-Use-parse_boolean-for-unredacted-content.patch b/packages/overlays/matrix-synapse/patches/0005-Use-parse_boolean-for-unredacted-content.patch deleted file mode 100644 index bfb3e75..0000000 --- a/packages/overlays/matrix-synapse/patches/0005-Use-parse_boolean-for-unredacted-content.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 9c3f28c68cb89e81a98561e0898b00c43a280a65 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Tue, 27 May 2025 06:14:26 +0200 -Subject: [PATCH 05/10] Use parse_boolean for unredacted content - ---- - synapse/rest/client/room.py | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py -index bb41575d46..63ea52baf8 100644 ---- a/synapse/rest/client/room.py -+++ b/synapse/rest/client/room.py -@@ -914,10 +914,9 @@ class RoomEventServlet(RestServlet): - requester = await self.auth.get_user_by_req(request, allow_guest=True) - - include_unredacted_content = self.msc2815_enabled and ( -- parse_string( -+ parse_boolean( - request, -- "fi.mau.msc2815.include_unredacted_content", -- allowed_values=("true", "false"), -+ "fi.mau.msc2815.include_unredacted_content" - ) - == "true" - ) --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0006-Expose-tombstone-in-room-admin-api.patch b/packages/overlays/matrix-synapse/patches/0006-Expose-tombstone-in-room-admin-api.patch deleted file mode 100644 index c12912e..0000000 --- a/packages/overlays/matrix-synapse/patches/0006-Expose-tombstone-in-room-admin-api.patch +++ /dev/null @@ -1,114 +0,0 @@ -From 23c042aead65385a500be6e671ddd0e942a7e864 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Tue, 27 May 2025 06:37:52 +0200 -Subject: [PATCH 06/10] Expose tombstone in room admin api - ---- - synapse/rest/admin/rooms.py | 5 ++++ - synapse/rest/client/room.py | 1 - - synapse/storage/databases/main/room.py | 36 +++++++++++++++++++++++++- - 3 files changed, 40 insertions(+), 2 deletions(-) - -diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py -index f8c5bf18d4..adac1f0362 100644 ---- a/synapse/rest/admin/rooms.py -+++ b/synapse/rest/admin/rooms.py -@@ -251,6 +251,10 @@ class ListRoomRestServlet(RestServlet): - direction = parse_enum(request, "dir", Direction, default=Direction.FORWARDS) - reverse_order = True if direction == Direction.BACKWARDS else False - -+ emma_include_tombstone = parse_boolean( -+ request, "emma_include_tombstone", default=False -+ ) -+ - # Return list of rooms according to parameters - rooms, total_rooms = await self.store.get_rooms_paginate( - start, -@@ -260,6 +264,7 @@ class ListRoomRestServlet(RestServlet): - search_term, - public_rooms, - empty_rooms, -+ emma_include_tombstone = emma_include_tombstone - ) - - response = { -diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py -index 63ea52baf8..38230de0de 100644 ---- a/synapse/rest/client/room.py -+++ b/synapse/rest/client/room.py -@@ -918,7 +918,6 @@ class RoomEventServlet(RestServlet): - request, - "fi.mau.msc2815.include_unredacted_content" - ) -- == "true" - ) - if include_unredacted_content and not await self.auth.is_server_admin( - requester -diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py -index 1df06a5171..347dbbba6b 100644 ---- a/synapse/storage/databases/main/room.py -+++ b/synapse/storage/databases/main/room.py -@@ -610,6 +610,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): - search_term: Optional[str], - public_rooms: Optional[bool], - empty_rooms: Optional[bool], -+ emma_include_tombstone: bool = False, - ) -> Tuple[List[Dict[str, Any]], int]: - """Function to retrieve a paginated list of rooms as json. - -@@ -629,6 +630,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): - If true, empty rooms are queried. - if false, empty rooms are excluded from the query. When it is - none (the default), both empty rooms and none-empty rooms are queried. -+ emma_include_tombstone: If true, include tombstone events in the results. - Returns: - A list of room dicts and an integer representing the total number of - rooms that exist given this query -@@ -797,11 +799,43 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): - room_count = cast(Tuple[int], txn.fetchone()) - return rooms, room_count[0] - -- return await self.db_pool.runInteraction( -+ result = await self.db_pool.runInteraction( - "get_rooms_paginate", - _get_rooms_paginate_txn, - ) - -+ if emma_include_tombstone: -+ room_id_sql, room_id_args = make_in_list_sql_clause( -+ self.database_engine, "cse.room_id", [r["room_id"] for r in result[0]] -+ ) -+ -+ tombstone_sql = """ -+ SELECT cse.room_id, cse.event_id, ej.json -+ FROM current_state_events cse -+ JOIN event_json ej USING (event_id) -+ WHERE cse.type = 'm.room.tombstone' -+ AND {room_id_sql} -+ """.format( -+ room_id_sql=room_id_sql -+ ) -+ -+ def _get_tombstones_txn( -+ txn: LoggingTransaction, -+ ) -> Dict[str, JsonDict]: -+ txn.execute(tombstone_sql, room_id_args) -+ for room_id, event_id, json in txn: -+ for result_room in result[0]: -+ if result_room["room_id"] == room_id: -+ result_room["gay.rory.synapse_admin_extensions.tombstone"] = db_to_json(json) -+ break -+ return result[0], result[1] -+ -+ result = await self.db_pool.runInteraction( -+ "get_rooms_tombstones", _get_tombstones_txn, -+ ) -+ -+ return result -+ - @cached(max_entries=10000) - async def get_ratelimit_for_user(self, user_id: str) -> Optional[RatelimitOverride]: - """Check if there are any overrides for ratelimiting for the given user --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch b/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch new file mode 100644 index 0000000..1c2841c --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch @@ -0,0 +1,29 @@ +From 07d72fd39ea3044577322647d5ed1dd8cb6f77d9 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Tue, 27 May 2025 06:14:26 +0200 +Subject: [PATCH 06/11] Use parse_boolean for unredacted content + +--- + synapse/rest/client/room.py | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py +index 4600a87778..e838341fde 100644 +--- a/synapse/rest/client/room.py ++++ b/synapse/rest/client/room.py +@@ -915,10 +915,9 @@ class RoomEventServlet(RestServlet): + requester = await self.auth.get_user_by_req(request, allow_guest=True) + + include_unredacted_content = self.msc2815_enabled and ( +- parse_string( ++ parse_boolean( + request, +- "fi.mau.msc2815.include_unredacted_content", +- allowed_values=("true", "false"), ++ "fi.mau.msc2815.include_unredacted_content" + ) + == "true" + ) +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch b/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch new file mode 100644 index 0000000..719705e --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch @@ -0,0 +1,114 @@ +From d3edb4aa9a225f521fdbc406c187fd40343b3963 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Tue, 27 May 2025 06:37:52 +0200 +Subject: [PATCH 07/11] Expose tombstone in room admin api + +--- + synapse/rest/admin/rooms.py | 5 ++++ + synapse/rest/client/room.py | 1 - + synapse/storage/databases/main/room.py | 36 +++++++++++++++++++++++++- + 3 files changed, 40 insertions(+), 2 deletions(-) + +diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py +index f8c5bf18d4..adac1f0362 100644 +--- a/synapse/rest/admin/rooms.py ++++ b/synapse/rest/admin/rooms.py +@@ -251,6 +251,10 @@ class ListRoomRestServlet(RestServlet): + direction = parse_enum(request, "dir", Direction, default=Direction.FORWARDS) + reverse_order = True if direction == Direction.BACKWARDS else False + ++ emma_include_tombstone = parse_boolean( ++ request, "emma_include_tombstone", default=False ++ ) ++ + # Return list of rooms according to parameters + rooms, total_rooms = await self.store.get_rooms_paginate( + start, +@@ -260,6 +264,7 @@ class ListRoomRestServlet(RestServlet): + search_term, + public_rooms, + empty_rooms, ++ emma_include_tombstone = emma_include_tombstone + ) + + response = { +diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py +index e838341fde..f61152c35b 100644 +--- a/synapse/rest/client/room.py ++++ b/synapse/rest/client/room.py +@@ -919,7 +919,6 @@ class RoomEventServlet(RestServlet): + request, + "fi.mau.msc2815.include_unredacted_content" + ) +- == "true" + ) + if include_unredacted_content and not await self.auth.is_server_admin( + requester +diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py +index 58451d3ff1..66b79db4ed 100644 +--- a/synapse/storage/databases/main/room.py ++++ b/synapse/storage/databases/main/room.py +@@ -610,6 +610,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): + search_term: Optional[str], + public_rooms: Optional[bool], + empty_rooms: Optional[bool], ++ emma_include_tombstone: bool = False, + ) -> Tuple[List[Dict[str, Any]], int]: + """Function to retrieve a paginated list of rooms as json. + +@@ -629,6 +630,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): + If true, empty rooms are queried. + if false, empty rooms are excluded from the query. When it is + none (the default), both empty rooms and none-empty rooms are queried. ++ emma_include_tombstone: If true, include tombstone events in the results. + Returns: + A list of room dicts and an integer representing the total number of + rooms that exist given this query +@@ -797,11 +799,43 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): + room_count = cast(Tuple[int], txn.fetchone()) + return rooms, room_count[0] + +- return await self.db_pool.runInteraction( ++ result = await self.db_pool.runInteraction( + "get_rooms_paginate", + _get_rooms_paginate_txn, + ) + ++ if emma_include_tombstone: ++ room_id_sql, room_id_args = make_in_list_sql_clause( ++ self.database_engine, "cse.room_id", [r["room_id"] for r in result[0]] ++ ) ++ ++ tombstone_sql = """ ++ SELECT cse.room_id, cse.event_id, ej.json ++ FROM current_state_events cse ++ JOIN event_json ej USING (event_id) ++ WHERE cse.type = 'm.room.tombstone' ++ AND {room_id_sql} ++ """.format( ++ room_id_sql=room_id_sql ++ ) ++ ++ def _get_tombstones_txn( ++ txn: LoggingTransaction, ++ ) -> Dict[str, JsonDict]: ++ txn.execute(tombstone_sql, room_id_args) ++ for room_id, event_id, json in txn: ++ for result_room in result[0]: ++ if result_room["room_id"] == room_id: ++ result_room["gay.rory.synapse_admin_extensions.tombstone"] = db_to_json(json) ++ break ++ return result[0], result[1] ++ ++ result = await self.db_pool.runInteraction( ++ "get_rooms_tombstones", _get_tombstones_txn, ++ ) ++ ++ return result ++ + @cached(max_entries=10000) + async def get_ratelimit_for_user(self, user_id: str) -> Optional[RatelimitOverride]: + """Check if there are any overrides for ratelimiting for the given user +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0007-fix-Always-recheck-messages-pagination-data-if-a-bac.patch b/packages/overlays/matrix-synapse/patches/0007-fix-Always-recheck-messages-pagination-data-if-a-bac.patch deleted file mode 100644 index 4ebc20c..0000000 --- a/packages/overlays/matrix-synapse/patches/0007-fix-Always-recheck-messages-pagination-data-if-a-bac.patch +++ /dev/null @@ -1,204 +0,0 @@ -From 99b146825a1a8257d05440ae3e331c68b8e1575a Mon Sep 17 00:00:00 2001 -From: Jason Little -Date: Wed, 30 Apr 2025 09:29:42 -0500 -Subject: [PATCH 07/10] fix: Always recheck `/messages` pagination data if a - backfill might have been needed (#28) - ---- - synapse/handlers/federation.py | 35 +++++++++++++-------------------- - synapse/handlers/pagination.py | 36 +++++++++++++++++++--------------- - 2 files changed, 34 insertions(+), 37 deletions(-) - -diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py -index a6de3e824d..ff751d25f6 100644 ---- a/synapse/handlers/federation.py -+++ b/synapse/handlers/federation.py -@@ -211,7 +211,7 @@ class FederationHandler: - @tag_args - async def maybe_backfill( - self, room_id: str, current_depth: int, limit: int, record_time: bool = True -- ) -> bool: -+ ) -> None: - """Checks the database to see if we should backfill before paginating, - and if so do. - -@@ -225,8 +225,6 @@ class FederationHandler: - should back paginate. - record_time: Whether to record the time it takes to backfill. - -- Returns: -- True if we actually tried to backfill something, otherwise False. - """ - # Starting the processing time here so we can include the room backfill - # linearizer lock queue in the timing -@@ -252,7 +250,7 @@ class FederationHandler: - limit: int, - *, - processing_start_time: Optional[int], -- ) -> bool: -+ ) -> None: - """ - Checks whether the `current_depth` is at or approaching any backfill - points in the room and if so, will backfill. We only care about -@@ -326,7 +324,7 @@ class FederationHandler: - limit=1, - ) - if not have_later_backfill_points: -- return False -+ return None - - logger.debug( - "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points." -@@ -346,15 +344,15 @@ class FederationHandler: - ) - # We return `False` because we're backfilling in the background and there is - # no new events immediately for the caller to know about yet. -- return False -+ return None - - # Even after recursing with `MAX_DEPTH`, we didn't find any - # backward extremities to backfill from. - if not sorted_backfill_points: - logger.debug( -- "_maybe_backfill_inner: Not backfilling as no backward extremeties found." -+ "_maybe_backfill_inner: Not backfilling as no backward extremities found." - ) -- return False -+ return None - - # If we're approaching an extremity we trigger a backfill, otherwise we - # no-op. -@@ -373,7 +371,7 @@ class FederationHandler: - current_depth, - limit, - ) -- return False -+ return None - - # For performance's sake, we only want to paginate from a particular extremity - # if we can actually see the events we'll get. Otherwise, we'd just spend a lot -@@ -441,7 +439,7 @@ class FederationHandler: - logger.debug( - "_maybe_backfill_inner: found no extremities which would be visible" - ) -- return False -+ return None - - logger.debug( - "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request -@@ -464,7 +462,7 @@ class FederationHandler: - ) - ) - -- async def try_backfill(domains: StrCollection) -> bool: -+ async def try_backfill(domains: StrCollection) -> None: - # TODO: Should we try multiple of these at a time? - - # Number of contacted remote homeservers that have denied our backfill -@@ -487,7 +485,7 @@ class FederationHandler: - # If this succeeded then we probably already have the - # appropriate stuff. - # TODO: We can probably do something more intelligent here. -- return True -+ return None - except NotRetryingDestination as e: - logger.info("_maybe_backfill_inner: %s", e) - continue -@@ -511,7 +509,7 @@ class FederationHandler: - ) - denied_count += 1 - if denied_count >= max_denied_count: -- return False -+ return None - continue - - logger.info("Failed to backfill from %s because %s", dom, e) -@@ -527,7 +525,7 @@ class FederationHandler: - ) - denied_count += 1 - if denied_count >= max_denied_count: -- return False -+ return None - continue - - logger.info("Failed to backfill from %s because %s", dom, e) -@@ -539,7 +537,7 @@ class FederationHandler: - logger.exception("Failed to backfill from %s because %s", dom, e) - continue - -- return False -+ return None - - # If we have the `processing_start_time`, then we can make an - # observation. We wouldn't have the `processing_start_time` in the case -@@ -551,14 +549,9 @@ class FederationHandler: - (processing_end_time - processing_start_time) / 1000 - ) - -- success = await try_backfill(likely_domains) -- if success: -- return True -- - # TODO: we could also try servers which were previously in the room, but - # are no longer. -- -- return False -+ return await try_backfill(likely_domains) - - async def send_invite(self, target_host: str, event: EventBase) -> EventBase: - """Sends the invite to the remote server for signing. -diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py -index 4070b74b7a..81cda38549 100644 ---- a/synapse/handlers/pagination.py -+++ b/synapse/handlers/pagination.py -@@ -577,27 +577,31 @@ class PaginationHandler: - or missing_too_many_events - or not_enough_events_to_fill_response - ): -- did_backfill = await self.hs.get_federation_handler().maybe_backfill( -+ # Historical Note: There used to be a check here for if backfill was -+ # successful or not -+ await self.hs.get_federation_handler().maybe_backfill( - room_id, - curr_topo, - limit=pagin_config.limit, - ) - -- # If we did backfill something, refetch the events from the database to -- # catch anything new that might have been added since we last fetched. -- if did_backfill: -- ( -- events, -- next_key, -- _, -- ) = await self.store.paginate_room_events_by_topological_ordering( -- room_id=room_id, -- from_key=from_token.room_key, -- to_key=to_room_key, -- direction=pagin_config.direction, -- limit=pagin_config.limit, -- event_filter=event_filter, -- ) -+ # Regardless if we backfilled or not, another worker or even a -+ # simultaneous request may have backfilled for us while we were held -+ # behind the linearizer. This should not have too much additional -+ # database load as it will only be triggered if a backfill *might* have -+ # been needed -+ ( -+ events, -+ next_key, -+ _, -+ ) = await self.store.paginate_room_events_by_topological_ordering( -+ room_id=room_id, -+ from_key=from_token.room_key, -+ to_key=to_room_key, -+ direction=pagin_config.direction, -+ limit=pagin_config.limit, -+ event_filter=event_filter, -+ ) - else: - # Otherwise, we can backfill in the background for eventual - # consistency's sake but we don't need to block the client waiting --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0008-Fix-pagination-with-large-gaps-of-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0008-Fix-pagination-with-large-gaps-of-rejected-events.patch deleted file mode 100644 index 81a6d3f..0000000 --- a/packages/overlays/matrix-synapse/patches/0008-Fix-pagination-with-large-gaps-of-rejected-events.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 6eb23d3018f68744ba363fb7a89a9a4982d67a19 Mon Sep 17 00:00:00 2001 -From: Nicolas Werner -Date: Sun, 8 Jun 2025 23:14:31 +0200 -Subject: [PATCH 08/10] Fix pagination with large gaps of rejected events - ---- - synapse/handlers/pagination.py | 13 +++++++++++-- - 1 file changed, 11 insertions(+), 2 deletions(-) - -diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py -index 81cda38549..365c9cabcb 100644 ---- a/synapse/handlers/pagination.py -+++ b/synapse/handlers/pagination.py -@@ -510,7 +510,7 @@ class PaginationHandler: - ( - events, - next_key, -- _, -+ limited, - ) = await self.store.paginate_room_events_by_topological_ordering( - room_id=room_id, - from_key=from_token.room_key, -@@ -593,7 +593,7 @@ class PaginationHandler: - ( - events, - next_key, -- _, -+ limited, - ) = await self.store.paginate_room_events_by_topological_ordering( - room_id=room_id, - from_key=from_token.room_key, -@@ -616,6 +616,15 @@ class PaginationHandler: - - next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key) - -+ # We might have hit some internal filtering first, for example rejected -+ # events. Ensure we return a pagination token then. -+ if not events and limited: -+ return { -+ "chunk": [], -+ "start": await from_token.to_string(self.store), -+ "end": await next_token.to_string(self.store), -+ } -+ - # if no events are returned from pagination, that implies - # we have reached the end of the available events. - # In that case we do not return end, to tell the client --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch b/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch new file mode 100644 index 0000000..363204e --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch @@ -0,0 +1,204 @@ +From afecddceaa6ece4cf797ce27e226a99acb8e8a6d Mon Sep 17 00:00:00 2001 +From: Jason Little +Date: Wed, 30 Apr 2025 09:29:42 -0500 +Subject: [PATCH 08/11] fix: Always recheck `/messages` pagination data if a + backfill might have been needed (#28) + +--- + synapse/handlers/federation.py | 35 +++++++++++++-------------------- + synapse/handlers/pagination.py | 36 +++++++++++++++++++--------------- + 2 files changed, 34 insertions(+), 37 deletions(-) + +diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py +index 015fb3edca..a0bc955121 100644 +--- a/synapse/handlers/federation.py ++++ b/synapse/handlers/federation.py +@@ -211,7 +211,7 @@ class FederationHandler: + @tag_args + async def maybe_backfill( + self, room_id: str, current_depth: int, limit: int, record_time: bool = True +- ) -> bool: ++ ) -> None: + """Checks the database to see if we should backfill before paginating, + and if so do. + +@@ -225,8 +225,6 @@ class FederationHandler: + should back paginate. + record_time: Whether to record the time it takes to backfill. + +- Returns: +- True if we actually tried to backfill something, otherwise False. + """ + # Starting the processing time here so we can include the room backfill + # linearizer lock queue in the timing +@@ -252,7 +250,7 @@ class FederationHandler: + limit: int, + *, + processing_start_time: Optional[int], +- ) -> bool: ++ ) -> None: + """ + Checks whether the `current_depth` is at or approaching any backfill + points in the room and if so, will backfill. We only care about +@@ -326,7 +324,7 @@ class FederationHandler: + limit=1, + ) + if not have_later_backfill_points: +- return False ++ return None + + logger.debug( + "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points." +@@ -346,15 +344,15 @@ class FederationHandler: + ) + # We return `False` because we're backfilling in the background and there is + # no new events immediately for the caller to know about yet. +- return False ++ return None + + # Even after recursing with `MAX_DEPTH`, we didn't find any + # backward extremities to backfill from. + if not sorted_backfill_points: + logger.debug( +- "_maybe_backfill_inner: Not backfilling as no backward extremeties found." ++ "_maybe_backfill_inner: Not backfilling as no backward extremities found." + ) +- return False ++ return None + + # If we're approaching an extremity we trigger a backfill, otherwise we + # no-op. +@@ -373,7 +371,7 @@ class FederationHandler: + current_depth, + limit, + ) +- return False ++ return None + + # For performance's sake, we only want to paginate from a particular extremity + # if we can actually see the events we'll get. Otherwise, we'd just spend a lot +@@ -441,7 +439,7 @@ class FederationHandler: + logger.debug( + "_maybe_backfill_inner: found no extremities which would be visible" + ) +- return False ++ return None + + logger.debug( + "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request +@@ -464,7 +462,7 @@ class FederationHandler: + ) + ) + +- async def try_backfill(domains: StrCollection) -> bool: ++ async def try_backfill(domains: StrCollection) -> None: + # TODO: Should we try multiple of these at a time? + + # Number of contacted remote homeservers that have denied our backfill +@@ -487,7 +485,7 @@ class FederationHandler: + # If this succeeded then we probably already have the + # appropriate stuff. + # TODO: We can probably do something more intelligent here. +- return True ++ return None + except NotRetryingDestination as e: + logger.info("_maybe_backfill_inner: %s", e) + continue +@@ -511,7 +509,7 @@ class FederationHandler: + ) + denied_count += 1 + if denied_count >= max_denied_count: +- return False ++ return None + continue + + logger.info("Failed to backfill from %s because %s", dom, e) +@@ -527,7 +525,7 @@ class FederationHandler: + ) + denied_count += 1 + if denied_count >= max_denied_count: +- return False ++ return None + continue + + logger.info("Failed to backfill from %s because %s", dom, e) +@@ -539,7 +537,7 @@ class FederationHandler: + logger.exception("Failed to backfill from %s because %s", dom, e) + continue + +- return False ++ return None + + # If we have the `processing_start_time`, then we can make an + # observation. We wouldn't have the `processing_start_time` in the case +@@ -551,14 +549,9 @@ class FederationHandler: + (processing_end_time - processing_start_time) / 1000 + ) + +- success = await try_backfill(likely_domains) +- if success: +- return True +- + # TODO: we could also try servers which were previously in the room, but + # are no longer. +- +- return False ++ return await try_backfill(likely_domains) + + async def send_invite(self, target_host: str, event: EventBase) -> EventBase: + """Sends the invite to the remote server for signing. +diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py +index 4070b74b7a..81cda38549 100644 +--- a/synapse/handlers/pagination.py ++++ b/synapse/handlers/pagination.py +@@ -577,27 +577,31 @@ class PaginationHandler: + or missing_too_many_events + or not_enough_events_to_fill_response + ): +- did_backfill = await self.hs.get_federation_handler().maybe_backfill( ++ # Historical Note: There used to be a check here for if backfill was ++ # successful or not ++ await self.hs.get_federation_handler().maybe_backfill( + room_id, + curr_topo, + limit=pagin_config.limit, + ) + +- # If we did backfill something, refetch the events from the database to +- # catch anything new that might have been added since we last fetched. +- if did_backfill: +- ( +- events, +- next_key, +- _, +- ) = await self.store.paginate_room_events_by_topological_ordering( +- room_id=room_id, +- from_key=from_token.room_key, +- to_key=to_room_key, +- direction=pagin_config.direction, +- limit=pagin_config.limit, +- event_filter=event_filter, +- ) ++ # Regardless if we backfilled or not, another worker or even a ++ # simultaneous request may have backfilled for us while we were held ++ # behind the linearizer. This should not have too much additional ++ # database load as it will only be triggered if a backfill *might* have ++ # been needed ++ ( ++ events, ++ next_key, ++ _, ++ ) = await self.store.paginate_room_events_by_topological_ordering( ++ room_id=room_id, ++ from_key=from_token.room_key, ++ to_key=to_room_key, ++ direction=pagin_config.direction, ++ limit=pagin_config.limit, ++ event_filter=event_filter, ++ ) + else: + # Otherwise, we can backfill in the background for eventual + # consistency's sake but we don't need to block the client waiting +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0009-Fix-nix-flake.patch b/packages/overlays/matrix-synapse/patches/0009-Fix-nix-flake.patch deleted file mode 100644 index 09a7f5c..0000000 --- a/packages/overlays/matrix-synapse/patches/0009-Fix-nix-flake.patch +++ /dev/null @@ -1,189 +0,0 @@ -From 69d854a8250493f9c1e468f24031378ff334cf7f Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Mon, 9 Jun 2025 17:38:34 +0200 -Subject: [PATCH 09/10] Fix nix flake - ---- - flake.lock | 58 +++++++++++++++++++----------------------------------- - flake.nix | 10 +++++++++- - 2 files changed, 29 insertions(+), 39 deletions(-) - -diff --git a/flake.lock b/flake.lock -index a6a2aea328..4e2f01153b 100644 ---- a/flake.lock -+++ b/flake.lock -@@ -39,15 +39,12 @@ - } - }, - "flake-utils": { -- "inputs": { -- "systems": "systems" -- }, - "locked": { -- "lastModified": 1685518550, -- "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", -+ "lastModified": 1667395993, -+ "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", -- "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", -+ "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" - }, - "original": { -@@ -152,27 +149,27 @@ - }, - "nixpkgs-stable": { - "locked": { -- "lastModified": 1685801374, -- "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", -+ "lastModified": 1678872516, -+ "narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=", - "owner": "NixOS", - "repo": "nixpkgs", -- "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", -+ "rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8", - "type": "github" - }, - "original": { - "owner": "NixOS", -- "ref": "nixos-23.05", -+ "ref": "nixos-22.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { -- "lastModified": 1729265718, -- "narHash": "sha256-4HQI+6LsO3kpWTYuVGIzhJs1cetFcwT7quWCk/6rqeo=", -+ "lastModified": 1748217807, -+ "narHash": "sha256-P3u2PXxMlo49PutQLnk2PhI/imC69hFl1yY4aT5Nax8=", - "owner": "NixOS", - "repo": "nixpkgs", -- "rev": "ccc0c2126893dd20963580b6478d1a10a4512185", -+ "rev": "3108eaa516ae22c2360928589731a4f1581526ef", - "type": "github" - }, - "original": { -@@ -184,11 +181,11 @@ - }, - "nixpkgs_3": { - "locked": { -- "lastModified": 1728538411, -- "narHash": "sha256-f0SBJz1eZ2yOuKUr5CA9BHULGXVSn6miBuUWdTyhUhU=", -+ "lastModified": 1744536153, -+ "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", - "owner": "NixOS", - "repo": "nixpkgs", -- "rev": "b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221", -+ "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", - "type": "github" - }, - "original": { -@@ -213,11 +210,11 @@ - "nixpkgs-stable": "nixpkgs-stable" - }, - "locked": { -- "lastModified": 1688056373, -- "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", -+ "lastModified": 1686050334, -+ "narHash": "sha256-R0mczWjDzBpIvM3XXhO908X5e2CQqjyh/gFbwZk/7/Q=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", -- "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", -+ "rev": "6881eb2ae5d8a3516e34714e7a90d9d95914c4dc", - "type": "github" - }, - "original": { -@@ -231,7 +228,7 @@ - "devenv": "devenv", - "nixpkgs": "nixpkgs_2", - "rust-overlay": "rust-overlay", -- "systems": "systems_2" -+ "systems": "systems" - } - }, - "rust-overlay": { -@@ -239,11 +236,11 @@ - "nixpkgs": "nixpkgs_3" - }, - "locked": { -- "lastModified": 1731897198, -- "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", -+ "lastModified": 1748313401, -+ "narHash": "sha256-x5UuDKP2Ui/TresAngUo9U4Ss9xfOmN8dAXU8OrkZmA=", - "owner": "oxalica", - "repo": "rust-overlay", -- "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", -+ "rev": "9c8ea175cf9af29edbcff121512e44092a8f37e4", - "type": "github" - }, - "original": { -@@ -266,21 +263,6 @@ - "repo": "default", - "type": "github" - } -- }, -- "systems_2": { -- "locked": { -- "lastModified": 1681028828, -- "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", -- "owner": "nix-systems", -- "repo": "default", -- "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", -- "type": "github" -- }, -- "original": { -- "owner": "nix-systems", -- "repo": "default", -- "type": "github" -- } - } - }, - "root": "root", -diff --git a/flake.nix b/flake.nix -index 749c10da1d..e33b233ece 100644 ---- a/flake.nix -+++ b/flake.nix -@@ -82,7 +82,7 @@ - # - # NOTE: We currently need to set the Rust version unnecessarily high - # in order to work around https://github.com/matrix-org/synapse/issues/15939 -- (rust-bin.stable."1.82.0".default.override { -+ (rust-bin.stable."1.87.0".default.override { - # Additionally install the "rust-src" extension to allow diving into the - # Rust source code in an IDE (rust-analyzer will also make use of it). - extensions = [ "rust-src" ]; -@@ -118,6 +118,8 @@ - # For releasing Synapse - debian-devscripts # (`dch` for manipulating the Debian changelog) - libnotify # (the release script uses `notify-send` to tell you when CI jobs are done) -+ -+ postgresql.pg_config - ]; - - # Install Python and manage a virtualenv with Poetry. -@@ -140,6 +142,9 @@ - # force compiling those binaries locally instead. - env.POETRY_INSTALLER_NO_BINARY = "ruff"; - -+ # Required to make git work -+ env.CARGO_NET_GIT_FETCH_WITH_CLI = "true"; -+ - # Install dependencies for the additional programming languages - # involved with Synapse development. - # -@@ -160,6 +165,9 @@ - services.postgres.initialDatabases = [ - { name = "synapse"; } - ]; -+ -+ services.postgres.port = 5433; -+ - # Create a postgres user called 'synapse_user' which has ownership - # over the 'synapse' database. - services.postgres.initialScript = '' --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch new file mode 100644 index 0000000..ebed62e --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch @@ -0,0 +1,50 @@ +From 2f2dd65326b8a8dc6b7ac99dbe7476abb2163469 Mon Sep 17 00:00:00 2001 +From: Nicolas Werner +Date: Sun, 8 Jun 2025 23:14:31 +0200 +Subject: [PATCH 09/11] Fix pagination with large gaps of rejected events + +--- + synapse/handlers/pagination.py | 13 +++++++++++-- + 1 file changed, 11 insertions(+), 2 deletions(-) + +diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py +index 81cda38549..365c9cabcb 100644 +--- a/synapse/handlers/pagination.py ++++ b/synapse/handlers/pagination.py +@@ -510,7 +510,7 @@ class PaginationHandler: + ( + events, + next_key, +- _, ++ limited, + ) = await self.store.paginate_room_events_by_topological_ordering( + room_id=room_id, + from_key=from_token.room_key, +@@ -593,7 +593,7 @@ class PaginationHandler: + ( + events, + next_key, +- _, ++ limited, + ) = await self.store.paginate_room_events_by_topological_ordering( + room_id=room_id, + from_key=from_token.room_key, +@@ -616,6 +616,15 @@ class PaginationHandler: + + next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key) + ++ # We might have hit some internal filtering first, for example rejected ++ # events. Ensure we return a pagination token then. ++ if not events and limited: ++ return { ++ "chunk": [], ++ "start": await from_token.to_string(self.store), ++ "end": await next_token.to_string(self.store), ++ } ++ + # if no events are returned from pagination, that implies + # we have reached the end of the available events. + # In that case we do not return end, to tell the client +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0010-Fix-gitignore-to-ignore-.venv.patch b/packages/overlays/matrix-synapse/patches/0010-Fix-gitignore-to-ignore-.venv.patch deleted file mode 100644 index abe0eb3..0000000 --- a/packages/overlays/matrix-synapse/patches/0010-Fix-gitignore-to-ignore-.venv.patch +++ /dev/null @@ -1,24 +0,0 @@ -From 35c7dfe6a8df912f14d4f18642e2af2675d607af Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Mon, 9 Jun 2025 17:46:10 +0200 -Subject: [PATCH 10/10] Fix gitignore to ignore .venv - ---- - .gitignore | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/.gitignore b/.gitignore -index a89f149ec1..0567934c4e 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -30,6 +30,7 @@ __pycache__/ - /*.signing.key - /env/ - /.venv*/ -+/.venv - /homeserver*.yaml - /logs - /media_store/ --- -2.49.0 - diff --git a/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch b/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch new file mode 100644 index 0000000..4df6090 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch @@ -0,0 +1,189 @@ +From 448de6ea7bfe1c6073726f517988e5deeb510861 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Mon, 9 Jun 2025 17:38:34 +0200 +Subject: [PATCH 10/11] Fix nix flake + +--- + flake.lock | 58 +++++++++++++++++++----------------------------------- + flake.nix | 10 +++++++++- + 2 files changed, 29 insertions(+), 39 deletions(-) + +diff --git a/flake.lock b/flake.lock +index a6a2aea328..4e2f01153b 100644 +--- a/flake.lock ++++ b/flake.lock +@@ -39,15 +39,12 @@ + } + }, + "flake-utils": { +- "inputs": { +- "systems": "systems" +- }, + "locked": { +- "lastModified": 1685518550, +- "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", ++ "lastModified": 1667395993, ++ "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", +- "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", ++ "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { +@@ -152,27 +149,27 @@ + }, + "nixpkgs-stable": { + "locked": { +- "lastModified": 1685801374, +- "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", ++ "lastModified": 1678872516, ++ "narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=", + "owner": "NixOS", + "repo": "nixpkgs", +- "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", ++ "rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8", + "type": "github" + }, + "original": { + "owner": "NixOS", +- "ref": "nixos-23.05", ++ "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { +- "lastModified": 1729265718, +- "narHash": "sha256-4HQI+6LsO3kpWTYuVGIzhJs1cetFcwT7quWCk/6rqeo=", ++ "lastModified": 1748217807, ++ "narHash": "sha256-P3u2PXxMlo49PutQLnk2PhI/imC69hFl1yY4aT5Nax8=", + "owner": "NixOS", + "repo": "nixpkgs", +- "rev": "ccc0c2126893dd20963580b6478d1a10a4512185", ++ "rev": "3108eaa516ae22c2360928589731a4f1581526ef", + "type": "github" + }, + "original": { +@@ -184,11 +181,11 @@ + }, + "nixpkgs_3": { + "locked": { +- "lastModified": 1728538411, +- "narHash": "sha256-f0SBJz1eZ2yOuKUr5CA9BHULGXVSn6miBuUWdTyhUhU=", ++ "lastModified": 1744536153, ++ "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", + "owner": "NixOS", + "repo": "nixpkgs", +- "rev": "b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221", ++ "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", + "type": "github" + }, + "original": { +@@ -213,11 +210,11 @@ + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { +- "lastModified": 1688056373, +- "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", ++ "lastModified": 1686050334, ++ "narHash": "sha256-R0mczWjDzBpIvM3XXhO908X5e2CQqjyh/gFbwZk/7/Q=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", +- "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", ++ "rev": "6881eb2ae5d8a3516e34714e7a90d9d95914c4dc", + "type": "github" + }, + "original": { +@@ -231,7 +228,7 @@ + "devenv": "devenv", + "nixpkgs": "nixpkgs_2", + "rust-overlay": "rust-overlay", +- "systems": "systems_2" ++ "systems": "systems" + } + }, + "rust-overlay": { +@@ -239,11 +236,11 @@ + "nixpkgs": "nixpkgs_3" + }, + "locked": { +- "lastModified": 1731897198, +- "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", ++ "lastModified": 1748313401, ++ "narHash": "sha256-x5UuDKP2Ui/TresAngUo9U4Ss9xfOmN8dAXU8OrkZmA=", + "owner": "oxalica", + "repo": "rust-overlay", +- "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", ++ "rev": "9c8ea175cf9af29edbcff121512e44092a8f37e4", + "type": "github" + }, + "original": { +@@ -266,21 +263,6 @@ + "repo": "default", + "type": "github" + } +- }, +- "systems_2": { +- "locked": { +- "lastModified": 1681028828, +- "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", +- "owner": "nix-systems", +- "repo": "default", +- "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", +- "type": "github" +- }, +- "original": { +- "owner": "nix-systems", +- "repo": "default", +- "type": "github" +- } + } + }, + "root": "root", +diff --git a/flake.nix b/flake.nix +index 4ff6518aed..76b3c1a4b0 100644 +--- a/flake.nix ++++ b/flake.nix +@@ -82,7 +82,7 @@ + # + # NOTE: We currently need to set the Rust version unnecessarily high + # in order to work around https://github.com/matrix-org/synapse/issues/15939 +- (rust-bin.stable."1.82.0".default.override { ++ (rust-bin.stable."1.87.0".default.override { + # Additionally install the "rust-src" extension to allow diving into the + # Rust source code in an IDE (rust-analyzer will also make use of it). + extensions = [ "rust-src" ]; +@@ -117,6 +117,8 @@ + # For releasing Synapse + debian-devscripts # (`dch` for manipulating the Debian changelog) + libnotify # (the release script uses `notify-send` to tell you when CI jobs are done) ++ ++ postgresql.pg_config + ]; + + # Install Python and manage a virtualenv with Poetry. +@@ -139,6 +141,9 @@ + # force compiling those binaries locally instead. + env.POETRY_INSTALLER_NO_BINARY = "ruff"; + ++ # Required to make git work ++ env.CARGO_NET_GIT_FETCH_WITH_CLI = "true"; ++ + # Install dependencies for the additional programming languages + # involved with Synapse development. + # +@@ -159,6 +164,9 @@ + services.postgres.initialDatabases = [ + { name = "synapse"; } + ]; ++ ++ services.postgres.port = 5433; ++ + # Create a postgres user called 'synapse_user' which has ownership + # over the 'synapse' database. + services.postgres.initialScript = '' +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch b/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch new file mode 100644 index 0000000..82335db --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch @@ -0,0 +1,24 @@ +From e1b50954048039a23c538cd260644ccc63d82941 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Mon, 9 Jun 2025 17:46:10 +0200 +Subject: [PATCH 11/11] Fix gitignore to ignore .venv + +--- + .gitignore | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/.gitignore b/.gitignore +index e333f2320b..3aec96e75e 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -30,6 +30,7 @@ __pycache__/ + /*.signing.key + /env/ + /.venv*/ ++/.venv + /homeserver*.yaml + /logs + /media_store/ +-- +2.49.0 + diff --git a/packages/overlays/matrix-synapse/patches/synapse-fast-links.patch b/packages/overlays/matrix-synapse/patches/synapse-fast-links.patch deleted file mode 100644 index c35ba87..0000000 --- a/packages/overlays/matrix-synapse/patches/synapse-fast-links.patch +++ /dev/null @@ -1,87 +0,0 @@ -diff --git a/synapse/storage/database.py b/synapse/storage/database.py -index cb4a585..1196781 100644 ---- a/synapse/storage/database.py -+++ b/synapse/storage/database.py -@@ -2503,6 +2503,49 @@ class DatabasePool: - - return txn.fetchall() - -+# requires database_engine.supports_using_any_list to be true -+def make_select_id_if_found_sql_clause( -+ database_engine: BaseDatabaseEngine, -+ column: str, -+ table: str, -+ iterable: Collection[Any], -+ *, -+ negative: bool = False, -+) -> Tuple[str, list]: -+ """Returns an SQL clause that checks the given column is in the iterable. -+ -+ On SQLite this expands to `column IN (?, ?, ...)`, whereas on Postgres -+ it expands to `column = ANY(?)`. While both DBs support the `IN` form, -+ using the `ANY` form on postgres means that it views queries with -+ different length iterables as the same, helping the query stats. -+ -+ Args: -+ database_engine -+ column: Name of the column -+ table: Name of the table -+ iterable: The values to check the column against. -+ negative: Whether we should check for inequality, i.e. `NOT IN` -+ -+ Returns: -+ A tuple of SQL query and the args -+ """ -+ # This should hopefully be faster, but also makes postgres query -+ # stats easier to understand. -+ if database_engine.supports_using_any_list: -+ if not negative: -+ clause = f"{column}_lookup AS {column} FROM UNNEST(?::bigint[]) {column}_lookup WHERE EXISTS(SELECT FROM {table} WHERE {column}={column}_lookup)" -+ else: -+ clause = f"{column}_lookup AS {column} FROM UNNEST(?::bigint[]) {column}_lookup WHERE NOT EXISTS(SELECT FROM {table} WHERE {column}={column}_lookup)" -+ -+ return clause, [list(iterable)] -+ else: -+ params = ",".join("?" for _ in iterable) -+ if not negative: -+ clause = f"DISTINCT {column} FROM {table} WHERE {column} IN ({params})" -+ else: -+ clause = f"DISTINCT {column} FROM {table} WHERE {column} NOT IN ({params})" -+ return clause, list(iterable) -+ - - def make_in_list_sql_clause( - database_engine: BaseDatabaseEngine, -diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py -index 46aa590..026f011 100644 ---- a/synapse/storage/databases/main/event_federation.py -+++ b/synapse/storage/databases/main/event_federation.py -@@ -52,6 +52,7 @@ from synapse.storage.database import ( - DatabasePool, - LoggingDatabaseConnection, - LoggingTransaction, -+ make_select_id_if_found_sql_clause, - ) - from synapse.storage.databases.main.events_worker import EventsWorkerStore - from synapse.storage.databases.main.signatures import SignatureWorkerStore -@@ -362,8 +363,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas - sql = """ - WITH RECURSIVE links(chain_id) AS ( - SELECT -- DISTINCT origin_chain_id -- FROM event_auth_chain_links WHERE %s -+ %s - UNION - SELECT - target_chain_id -@@ -380,8 +380,8 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas - while chains_to_fetch: - batch2 = tuple(itertools.islice(chains_to_fetch, 1000)) - chains_to_fetch.difference_update(batch2) -- clause, args = make_in_list_sql_clause( -- txn.database_engine, "origin_chain_id", batch2 -+ clause, args = make_select_id_if_found_sql_clause( -+ txn.database_engine, "origin_chain_id", "event_auth_chain_links", batch2 - ) - txn.execute(sql % (clause,), args) - -- cgit 1.5.1