diff options
| -rw-r--r-- | changelog.d/4715.misc | 1 | ||||
| -rw-r--r-- | debian/changelog | 6 | ||||
| -rw-r--r-- | debian/install | 1 | ||||
| -rwxr-xr-x | debian/manage_debconf.pl | 130 | ||||
| -rwxr-xr-x | debian/matrix-synapse-py3.config (renamed from debian/config) | 3 | ||||
| -rw-r--r-- | debian/matrix-synapse-py3.postinst | 33 | ||||
| -rw-r--r-- | synapse/replication/slave/storage/_base.py | 7 | ||||
| -rw-r--r-- | synapse/storage/_base.py | 40 |
8 files changed, 201 insertions, 20 deletions
diff --git a/changelog.d/4715.misc b/changelog.d/4715.misc new file mode 100644 index 0000000000..4dc18378e7 --- /dev/null +++ b/changelog.d/4715.misc @@ -0,0 +1 @@ +Improve replication performance by reducing cache invalidation traffic. diff --git a/debian/changelog b/debian/changelog index 124128920b..7631406a68 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (0.99.2) UNRELEASED; urgency=medium + + * Fix overwriting of config settings on upgrade. + + -- Synapse Packaging team <packages@matrix.org> Wed, 20 Feb 2019 17:11:25 +0000 + matrix-synapse-py3 (0.99.1.1) stable; urgency=medium * New synapse release 0.99.1.1 diff --git a/debian/install b/debian/install index 3d916a9718..43dc8c6904 100644 --- a/debian/install +++ b/debian/install @@ -1 +1,2 @@ debian/log.yaml etc/matrix-synapse +debian/manage_debconf.pl /opt/venvs/matrix-synapse/lib/ diff --git a/debian/manage_debconf.pl b/debian/manage_debconf.pl new file mode 100755 index 0000000000..be8ed32050 --- /dev/null +++ b/debian/manage_debconf.pl @@ -0,0 +1,130 @@ +#!/usr/bin/perl +# +# Interface between our config files and the debconf database. +# +# Usage: +# +# manage_debconf.pl <action> +# +# where <action> can be: +# +# read: read the configuration from the yaml into debconf +# update: update the yaml config according to the debconf database +use strict; +use warnings; + +use Debconf::Client::ConfModule (qw/get set/); + +# map from the name of a setting in our .yaml file to the relevant debconf +# setting. +my %MAPPINGS=( + server_name => 'matrix-synapse/server-name', + report_stats => 'matrix-synapse/report-stats', +); + +# enable debug if dpkg --debug +my $DEBUG = $ENV{DPKG_MAINTSCRIPT_DEBUG}; + +sub read_config { + my @files = @_; + + foreach my $file (@files) { + print STDERR "reading $file\n" if $DEBUG; + + open my $FH, "<", $file or next; + + # rudimentary parsing which (a) avoids having to depend on a yaml library, + # and (b) is tolerant of yaml errors + while($_ = <$FH>) { + while (my ($setting, $debconf) = each %MAPPINGS) { + $setting = quotemeta $setting; + if(/^${setting}\s*:(.*)$/) { + my $val = $1; + + # remove leading/trailing whitespace + $val =~ s/^\s*//; + $val =~ s/\s*$//; + + # remove surrounding quotes + if ($val =~ /^"(.*)"$/ || $val =~ /^'(.*)'$/) { + $val = $1; + } + + print STDERR ">> $debconf = $val\n" if $DEBUG; + set($debconf, $val); + } + } + } + close $FH; + } +} + +sub update_config { + my @files = @_; + + my %substs = (); + while (my ($setting, $debconf) = each %MAPPINGS) { + my @res = get($debconf); + $substs{$setting} = $res[1] if $res[0] == 0; + } + + foreach my $file (@files) { + print STDERR "checking $file\n" if $DEBUG; + + open my $FH, "<", $file or next; + + my $updated = 0; + + # read the whole file into memory + my @lines = <$FH>; + + while (my ($setting, $val) = each %substs) { + $setting = quotemeta $setting; + + map { + if (/^${setting}\s*:\s*(.*)\s*$/) { + my $current = $1; + if ($val ne $current) { + $_ = "${setting}: $val\n"; + $updated = 1; + } + } + } @lines; + } + close $FH; + + next unless $updated; + + print STDERR "updating $file\n" if $DEBUG; + open $FH, ">", $file or die "unable to update $file"; + print $FH @lines; + close $FH; + } +} + + +my $cmd = $ARGV[0]; + +my $read = 0; +my $update = 0; + +if (not $cmd) { + die "must specify a command to perform\n"; +} elsif ($cmd eq 'read') { + $read = 1; +} elsif ($cmd eq 'update') { + $update = 1; +} else { + die "unknown command '$cmd'\n"; +} + +my @files = ( + "/etc/matrix-synapse/homeserver.yaml", + glob("/etc/matrix-synapse/conf.d/*.yaml"), +); + +if ($read) { + read_config(@files); +} elsif ($update) { + update_config(@files); +} diff --git a/debian/config b/debian/matrix-synapse-py3.config index 9fb6913298..3bda3292f1 100755 --- a/debian/config +++ b/debian/matrix-synapse-py3.config @@ -4,6 +4,9 @@ set -e . /usr/share/debconf/confmodule +# try to update the debconf db according to whatever is in the config files +/opt/venvs/matrix-synapse/lib/manage_debconf.pl read || true + db_input high matrix-synapse/server-name || true db_input high matrix-synapse/report-stats || true db_go diff --git a/debian/matrix-synapse-py3.postinst b/debian/matrix-synapse-py3.postinst index 0509acd0a4..c0dd7e5534 100644 --- a/debian/matrix-synapse-py3.postinst +++ b/debian/matrix-synapse-py3.postinst @@ -8,19 +8,36 @@ USER="matrix-synapse" case "$1" in configure|reconfigure) - # Set server name in config file - mkdir -p "/etc/matrix-synapse/conf.d/" - db_get matrix-synapse/server-name - if [ "$RET" ]; then - echo "server_name: $RET" > $CONFIGFILE_SERVERNAME + # generate template config files if they don't exist + mkdir -p "/etc/matrix-synapse/conf.d/" + if [ ! -e "$CONFIGFILE_SERVERNAME" ]; then + cat > "$CONFIGFILE_SERVERNAME" <<EOF +# This file is autogenerated, and will be recreated on upgrade if it is deleted. +# Any changes you make will be preserved. + +# The domain name of the server, with optional explicit port. +# This is used by remote servers to connect to this server, +# e.g. matrix.org, localhost:8080, etc. +# This is also the last part of your UserID. +# +server_name: '' +EOF fi - db_get matrix-synapse/report-stats - if [ "$RET" ]; then - echo "report_stats: $RET" > $CONFIGFILE_REPORTSTATS + if [ ! -e "$CONFIGFILE_REPORTSTATS" ]; then + cat > "$CONFIGFILE_REPORTSTATS" <<EOF +# This file is autogenerated, and will be recreated on upgrade if it is deleted. +# Any changes you make will be preserved. + +# Whether to report anonymized homeserver usage statistics. +report_stats: false +EOF fi + # update the config files according to whatever is in the debconf database + /opt/venvs/matrix-synapse/lib/manage_debconf.pl update + if ! getent passwd $USER >/dev/null; then adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER fi diff --git a/synapse/replication/slave/storage/_base.py b/synapse/replication/slave/storage/_base.py index 1353a32d00..817d1f67f9 100644 --- a/synapse/replication/slave/storage/_base.py +++ b/synapse/replication/slave/storage/_base.py @@ -59,12 +59,7 @@ class BaseSlavedStore(SQLBaseStore): members_changed = set(row.keys[1:]) self._invalidate_state_caches(room_id, members_changed) else: - try: - getattr(self, row.cache_func).invalidate(tuple(row.keys)) - except AttributeError: - # We probably haven't pulled in the cache in this worker, - # which is fine. - pass + self._attempt_to_invalidate_cache(row.cache_func, tuple(row.keys)) def _invalidate_cache_and_stream(self, txn, cache_func, keys): txn.call_after(cache_func.invalidate, keys) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3d895da43c..5a80eef211 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1342,15 +1342,43 @@ class SQLBaseStore(object): changed """ for member in members_changed: - self.get_rooms_for_user_with_stream_ordering.invalidate((member,)) + self._attempt_to_invalidate_cache( + "get_rooms_for_user_with_stream_ordering", (member,), + ) for host in set(get_domain_from_id(u) for u in members_changed): - self.is_host_joined.invalidate((room_id, host)) - self.was_host_joined.invalidate((room_id, host)) + self._attempt_to_invalidate_cache( + "is_host_joined", (room_id, host,), + ) + self._attempt_to_invalidate_cache( + "was_host_joined", (room_id, host,), + ) + + self._attempt_to_invalidate_cache( + "get_users_in_room", (room_id,), + ) + self._attempt_to_invalidate_cache( + "get_room_summary", (room_id,), + ) + self._attempt_to_invalidate_cache( + "get_current_state_ids", (room_id,), + ) + + def _attempt_to_invalidate_cache(self, cache_name, key): + """Attempts to invalidate the cache of the given name, ignoring if the + cache doesn't exist. Mainly used for invalidating caches on workers, + where they may not have the cache. - self.get_users_in_room.invalidate((room_id,)) - self.get_room_summary.invalidate((room_id,)) - self.get_current_state_ids.invalidate((room_id,)) + Args: + cache_name (str) + key (tuple) + """ + try: + getattr(self, cache_name).invalidate(key) + except AttributeError: + # We probably haven't pulled in the cache in this worker, + # which is fine. + pass def _send_invalidation_to_replication(self, txn, cache_name, keys): """Notifies replication that given cache has been invalidated. |