diff --git a/changelog.d/4715.misc b/changelog.d/4715.misc
new file mode 100644
index 0000000000..4dc18378e7
--- /dev/null
+++ b/changelog.d/4715.misc
@@ -0,0 +1 @@
+Improve replication performance by reducing cache invalidation traffic.
diff --git a/debian/changelog b/debian/changelog
index 124128920b..7631406a68 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (0.99.2) UNRELEASED; urgency=medium
+
+ * Fix overwriting of config settings on upgrade.
+
+ -- Synapse Packaging team <packages@matrix.org> Wed, 20 Feb 2019 17:11:25 +0000
+
matrix-synapse-py3 (0.99.1.1) stable; urgency=medium
* New synapse release 0.99.1.1
diff --git a/debian/install b/debian/install
index 3d916a9718..43dc8c6904 100644
--- a/debian/install
+++ b/debian/install
@@ -1 +1,2 @@
debian/log.yaml etc/matrix-synapse
+debian/manage_debconf.pl /opt/venvs/matrix-synapse/lib/
diff --git a/debian/manage_debconf.pl b/debian/manage_debconf.pl
new file mode 100755
index 0000000000..be8ed32050
--- /dev/null
+++ b/debian/manage_debconf.pl
@@ -0,0 +1,130 @@
+#!/usr/bin/perl
+#
+# Interface between our config files and the debconf database.
+#
+# Usage:
+#
+# manage_debconf.pl <action>
+#
+# where <action> can be:
+#
+# read: read the configuration from the yaml into debconf
+# update: update the yaml config according to the debconf database
+use strict;
+use warnings;
+
+use Debconf::Client::ConfModule (qw/get set/);
+
+# map from the name of a setting in our .yaml file to the relevant debconf
+# setting.
+my %MAPPINGS=(
+ server_name => 'matrix-synapse/server-name',
+ report_stats => 'matrix-synapse/report-stats',
+);
+
+# enable debug if dpkg --debug
+my $DEBUG = $ENV{DPKG_MAINTSCRIPT_DEBUG};
+
+sub read_config {
+ my @files = @_;
+
+ foreach my $file (@files) {
+ print STDERR "reading $file\n" if $DEBUG;
+
+ open my $FH, "<", $file or next;
+
+ # rudimentary parsing which (a) avoids having to depend on a yaml library,
+ # and (b) is tolerant of yaml errors
+ while($_ = <$FH>) {
+ while (my ($setting, $debconf) = each %MAPPINGS) {
+ $setting = quotemeta $setting;
+ if(/^${setting}\s*:(.*)$/) {
+ my $val = $1;
+
+ # remove leading/trailing whitespace
+ $val =~ s/^\s*//;
+ $val =~ s/\s*$//;
+
+ # remove surrounding quotes
+ if ($val =~ /^"(.*)"$/ || $val =~ /^'(.*)'$/) {
+ $val = $1;
+ }
+
+ print STDERR ">> $debconf = $val\n" if $DEBUG;
+ set($debconf, $val);
+ }
+ }
+ }
+ close $FH;
+ }
+}
+
+sub update_config {
+ my @files = @_;
+
+ my %substs = ();
+ while (my ($setting, $debconf) = each %MAPPINGS) {
+ my @res = get($debconf);
+ $substs{$setting} = $res[1] if $res[0] == 0;
+ }
+
+ foreach my $file (@files) {
+ print STDERR "checking $file\n" if $DEBUG;
+
+ open my $FH, "<", $file or next;
+
+ my $updated = 0;
+
+ # read the whole file into memory
+ my @lines = <$FH>;
+
+ while (my ($setting, $val) = each %substs) {
+ $setting = quotemeta $setting;
+
+ map {
+ if (/^${setting}\s*:\s*(.*)\s*$/) {
+ my $current = $1;
+ if ($val ne $current) {
+ $_ = "${setting}: $val\n";
+ $updated = 1;
+ }
+ }
+ } @lines;
+ }
+ close $FH;
+
+ next unless $updated;
+
+ print STDERR "updating $file\n" if $DEBUG;
+ open $FH, ">", $file or die "unable to update $file";
+ print $FH @lines;
+ close $FH;
+ }
+}
+
+
+my $cmd = $ARGV[0];
+
+my $read = 0;
+my $update = 0;
+
+if (not $cmd) {
+ die "must specify a command to perform\n";
+} elsif ($cmd eq 'read') {
+ $read = 1;
+} elsif ($cmd eq 'update') {
+ $update = 1;
+} else {
+ die "unknown command '$cmd'\n";
+}
+
+my @files = (
+ "/etc/matrix-synapse/homeserver.yaml",
+ glob("/etc/matrix-synapse/conf.d/*.yaml"),
+);
+
+if ($read) {
+ read_config(@files);
+} elsif ($update) {
+ update_config(@files);
+}
diff --git a/debian/config b/debian/matrix-synapse-py3.config
index 9fb6913298..3bda3292f1 100755
--- a/debian/config
+++ b/debian/matrix-synapse-py3.config
@@ -4,6 +4,9 @@ set -e
. /usr/share/debconf/confmodule
+# try to update the debconf db according to whatever is in the config files
+/opt/venvs/matrix-synapse/lib/manage_debconf.pl read || true
+
db_input high matrix-synapse/server-name || true
db_input high matrix-synapse/report-stats || true
db_go
diff --git a/debian/matrix-synapse-py3.postinst b/debian/matrix-synapse-py3.postinst
index 0509acd0a4..c0dd7e5534 100644
--- a/debian/matrix-synapse-py3.postinst
+++ b/debian/matrix-synapse-py3.postinst
@@ -8,19 +8,36 @@ USER="matrix-synapse"
case "$1" in
configure|reconfigure)
- # Set server name in config file
- mkdir -p "/etc/matrix-synapse/conf.d/"
- db_get matrix-synapse/server-name
- if [ "$RET" ]; then
- echo "server_name: $RET" > $CONFIGFILE_SERVERNAME
+ # generate template config files if they don't exist
+ mkdir -p "/etc/matrix-synapse/conf.d/"
+ if [ ! -e "$CONFIGFILE_SERVERNAME" ]; then
+ cat > "$CONFIGFILE_SERVERNAME" <<EOF
+# This file is autogenerated, and will be recreated on upgrade if it is deleted.
+# Any changes you make will be preserved.
+
+# The domain name of the server, with optional explicit port.
+# This is used by remote servers to connect to this server,
+# e.g. matrix.org, localhost:8080, etc.
+# This is also the last part of your UserID.
+#
+server_name: ''
+EOF
fi
- db_get matrix-synapse/report-stats
- if [ "$RET" ]; then
- echo "report_stats: $RET" > $CONFIGFILE_REPORTSTATS
+ if [ ! -e "$CONFIGFILE_REPORTSTATS" ]; then
+ cat > "$CONFIGFILE_REPORTSTATS" <<EOF
+# This file is autogenerated, and will be recreated on upgrade if it is deleted.
+# Any changes you make will be preserved.
+
+# Whether to report anonymized homeserver usage statistics.
+report_stats: false
+EOF
fi
+ # update the config files according to whatever is in the debconf database
+ /opt/venvs/matrix-synapse/lib/manage_debconf.pl update
+
if ! getent passwd $USER >/dev/null; then
adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER
fi
diff --git a/synapse/replication/slave/storage/_base.py b/synapse/replication/slave/storage/_base.py
index 1353a32d00..817d1f67f9 100644
--- a/synapse/replication/slave/storage/_base.py
+++ b/synapse/replication/slave/storage/_base.py
@@ -59,12 +59,7 @@ class BaseSlavedStore(SQLBaseStore):
members_changed = set(row.keys[1:])
self._invalidate_state_caches(room_id, members_changed)
else:
- try:
- getattr(self, row.cache_func).invalidate(tuple(row.keys))
- except AttributeError:
- # We probably haven't pulled in the cache in this worker,
- # which is fine.
- pass
+ self._attempt_to_invalidate_cache(row.cache_func, tuple(row.keys))
def _invalidate_cache_and_stream(self, txn, cache_func, keys):
txn.call_after(cache_func.invalidate, keys)
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 3d895da43c..5a80eef211 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -1342,15 +1342,43 @@ class SQLBaseStore(object):
changed
"""
for member in members_changed:
- self.get_rooms_for_user_with_stream_ordering.invalidate((member,))
+ self._attempt_to_invalidate_cache(
+ "get_rooms_for_user_with_stream_ordering", (member,),
+ )
for host in set(get_domain_from_id(u) for u in members_changed):
- self.is_host_joined.invalidate((room_id, host))
- self.was_host_joined.invalidate((room_id, host))
+ self._attempt_to_invalidate_cache(
+ "is_host_joined", (room_id, host,),
+ )
+ self._attempt_to_invalidate_cache(
+ "was_host_joined", (room_id, host,),
+ )
+
+ self._attempt_to_invalidate_cache(
+ "get_users_in_room", (room_id,),
+ )
+ self._attempt_to_invalidate_cache(
+ "get_room_summary", (room_id,),
+ )
+ self._attempt_to_invalidate_cache(
+ "get_current_state_ids", (room_id,),
+ )
+
+ def _attempt_to_invalidate_cache(self, cache_name, key):
+ """Attempts to invalidate the cache of the given name, ignoring if the
+ cache doesn't exist. Mainly used for invalidating caches on workers,
+ where they may not have the cache.
- self.get_users_in_room.invalidate((room_id,))
- self.get_room_summary.invalidate((room_id,))
- self.get_current_state_ids.invalidate((room_id,))
+ Args:
+ cache_name (str)
+ key (tuple)
+ """
+ try:
+ getattr(self, cache_name).invalidate(key)
+ except AttributeError:
+ # We probably haven't pulled in the cache in this worker,
+ # which is fine.
+ pass
def _send_invalidation_to_replication(self, txn, cache_name, keys):
"""Notifies replication that given cache has been invalidated.
|
