From 8503dd0047119caa5b98a3fd56ac2b14dd09af0b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 7 Jun 2018 16:03:16 +0100 Subject: Remove event re-signing hacks These "temporary fixes" have been here three and a half years, and I can't find any events in the matrix.org database where the calculated signature differs from what's in the db. It's time for them to go away. --- synapse/handlers/federation.py | 43 ------------------------------------------ 1 file changed, 43 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index fcf94befb7..60b97b140e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -938,16 +938,6 @@ class FederationHandler(BaseHandler): [auth_id for auth_id, _ in event.auth_events], include_given=True ) - - for event in auth: - event.signatures.update( - compute_event_signature( - event, - self.hs.hostname, - self.hs.config.signing_key[0] - ) - ) - defer.returnValue([e for e in auth]) @log_function @@ -1405,18 +1395,6 @@ class FederationHandler(BaseHandler): del results[(event.type, event.state_key)] res = list(results.values()) - for event in res: - # We sign these again because there was a bug where we - # incorrectly signed things the first time round - if self.is_mine_id(event.event_id): - event.signatures.update( - compute_event_signature( - event, - self.hs.hostname, - self.hs.config.signing_key[0] - ) - ) - defer.returnValue(res) else: defer.returnValue([]) @@ -1481,18 +1459,6 @@ class FederationHandler(BaseHandler): ) if event: - if self.is_mine_id(event.event_id): - # FIXME: This is a temporary work around where we occasionally - # return events slightly differently than when they were - # originally signed - event.signatures.update( - compute_event_signature( - event, - self.hs.hostname, - self.hs.config.signing_key[0] - ) - ) - if do_auth: in_room = yield self.auth.check_host_in_room( event.room_id, @@ -1760,15 +1726,6 @@ class FederationHandler(BaseHandler): local_auth_chain, remote_auth_chain ) - for event in ret["auth_chain"]: - event.signatures.update( - compute_event_signature( - event, - self.hs.hostname, - self.hs.config.signing_key[0] - ) - ) - logger.debug("on_query_auth returning: %s", ret) defer.returnValue(ret) -- cgit 1.5.1 From c4e7ad0e0fbc27401d021a396579cfdefee5dbf9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 4 Jul 2018 09:15:45 +0100 Subject: Add changelog --- changelog.d/3367.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3367.misc diff --git a/changelog.d/3367.misc b/changelog.d/3367.misc new file mode 100644 index 0000000000..1f21ddea48 --- /dev/null +++ b/changelog.d/3367.misc @@ -0,0 +1 @@ +Remove unnecessary event re-signing hacks \ No newline at end of file -- cgit 1.5.1 From 36f4fd3e1edfa6032a5ace89e38ec7268e06dee6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 11 Jul 2018 15:45:44 +0100 Subject: Comment dummy TURN parameters in default config This default config is parsed and used a base before the actual config is overlaid, so with these values not commented out, the code to detect when no turn params were set and refuse to generate credentials was never firing because the dummy default was always set. --- changelog.d/3511.bugfix | 1 + synapse/config/voip.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/3511.bugfix diff --git a/changelog.d/3511.bugfix b/changelog.d/3511.bugfix new file mode 100644 index 0000000000..460fe24ac9 --- /dev/null +++ b/changelog.d/3511.bugfix @@ -0,0 +1 @@ +Don't generate TURN credentials if no TURN config options are set diff --git a/synapse/config/voip.py b/synapse/config/voip.py index 3a4e16fa96..d07bd24ffd 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -30,10 +30,10 @@ class VoipConfig(Config): ## Turn ## # The public URIs of the TURN server to give to clients - turn_uris: [] + #turn_uris: [] # The shared secret used to compute passwords for the TURN server - turn_shared_secret: "YOUR_SHARED_SECRET" + #turn_shared_secret: "YOUR_SHARED_SECRET" # The Username and password if the TURN server needs them and # does not use a token -- cgit 1.5.1 From 1b5425527cb514499cbba7d6fe06bde7d07d3ec2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 11 Jul 2018 16:02:29 +0100 Subject: I failed to correctly guess the PR number --- changelog.d/3511.bugfix | 1 - changelog.d/3514.bugfix | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/3511.bugfix create mode 100644 changelog.d/3514.bugfix diff --git a/changelog.d/3511.bugfix b/changelog.d/3511.bugfix deleted file mode 100644 index 460fe24ac9..0000000000 --- a/changelog.d/3511.bugfix +++ /dev/null @@ -1 +0,0 @@ -Don't generate TURN credentials if no TURN config options are set diff --git a/changelog.d/3514.bugfix b/changelog.d/3514.bugfix new file mode 100644 index 0000000000..460fe24ac9 --- /dev/null +++ b/changelog.d/3514.bugfix @@ -0,0 +1 @@ +Don't generate TURN credentials if no TURN config options are set -- cgit 1.5.1 From 6e3fc657b4c8db10e872ba1c7264e82b7fed5ed0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 10:50:33 +0100 Subject: Resource tracking for background processes This introduces a mechanism for tracking resource usage by background processes, along with an example of how it will be used. This will help address #3518, but more importantly will give us better insights into things which are happening but not being shown up by the request metrics. We *could* do this with Measure blocks, but: - I think having them pulled out as a completely separate metric class will make it easier to distinguish top-level processes from those which are nested. - I want to be able to report on in-flight background processes, and I don't think we want to do this for *all* Measure blocks. --- synapse/federation/transaction_queue.py | 12 +- synapse/metrics/background_process_metrics.py | 179 ++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 synapse/metrics/background_process_metrics.py diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 5a956ecfb3..5c5a73b73c 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -30,7 +30,8 @@ from synapse.metrics import ( sent_edus_counter, sent_transactions_counter, ) -from synapse.util import PreserveLoggingContext, logcontext +from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.util import logcontext from synapse.util.metrics import measure_func from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter @@ -165,10 +166,11 @@ class TransactionQueue(object): if self._is_processing: return - # fire off a processing loop in the background. It's likely it will - # outlast the current request, so run it in the sentinel logcontext. - with PreserveLoggingContext(): - self._process_event_queue_loop() + # fire off a processing loop in the background + run_as_background_process( + "process_transaction_queue", + self._process_event_queue_loop, + ) @defer.inlineCallbacks def _process_event_queue_loop(self): diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py new file mode 100644 index 0000000000..9d820e44a6 --- /dev/null +++ b/synapse/metrics/background_process_metrics.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +from prometheus_client.core import REGISTRY, Counter, GaugeMetricFamily + +from twisted.internet import defer + +from synapse.util.logcontext import LoggingContext, PreserveLoggingContext + +_background_process_start_count = Counter( + "synapse_background_process_start_count", + "Number of background processes started", + ["name"], +) + +# we set registry=None in all of these to stop them getting registered with +# the default registry. Instead we collect them all via the CustomCollector, +# which ensures that we can update them before they are collected. +# +_background_process_ru_utime = Counter( + "synapse_background_process_ru_utime_seconds", + "User CPU time used by background processes, in seconds", + ["name"], + registry=None, +) + +_background_process_ru_stime = Counter( + "synapse_background_process_ru_stime_seconds", + "System CPU time used by background processes, in seconds", + ["name"], + registry=None, +) + +_background_process_db_txn_count = Counter( + "synapse_background_process_db_txn_count", + "Number of database transactions done by background processes", + ["name"], + registry=None, +) + +_background_process_db_txn_duration = Counter( + "synapse_background_process_db_txn_duration_seconds", + ("Seconds spent by background processes waiting for database " + "transactions, excluding scheduling time"), + ["name"], + registry=None, +) + +_background_process_db_sched_duration = Counter( + "synapse_background_process_db_sched_duration_seconds", + "Seconds spent by background processes waiting for database connections", + ["name"], + registry=None, +) + +# map from description to a counter, so that we can name our logcontexts +# incrementally. (It actually duplicates _background_process_start_count, but +# it's much simpler to do so than to try to combine them.) +_background_process_counts = dict() # type: dict[str, int] + +# map from description to the currently running background processes. +# +# it's kept as a dict of sets rather than a big set so that we can keep track +# of process descriptions that no longer have any active processes. +_background_processes = dict() # type: dict[str, set[_BackgroundProcess]] + + +class _Collector(object): + """A custom metrics collector for the background process metrics. + + Ensures that all of the metrics are up-to-date with any in-flight processes + before they are returned. + """ + def collect(self): + background_process_in_flight_count = GaugeMetricFamily( + "synapse_background_process_in_flight_count", + "Number of background processes in flight", + labels=["name"], + ) + + for desc, processes in six.iteritems(_background_processes): + background_process_in_flight_count.add_metric( + (desc,), len(processes), + ) + for process in processes: + process.update_metrics() + + yield background_process_in_flight_count + + # now we need to run collect() over each of the static Counters, and + # yield each metric they return. + for m in ( + _background_process_ru_utime, + _background_process_ru_stime, + _background_process_db_txn_count, + _background_process_db_txn_duration, + _background_process_db_sched_duration, + ): + for r in m.collect(): + yield r + + +REGISTRY.register(_Collector()) + + +class _BackgroundProcess(object): + def __init__(self, desc, ctx): + self.desc = desc + self._context = ctx + self._reported_stats = None + + def update_metrics(self): + """Updates the metrics with values from this process.""" + new_stats = self._context.get_resource_usage() + if self._reported_stats is None: + diff = new_stats + else: + diff = new_stats - self._reported_stats + self._reported_stats = new_stats + + _background_process_ru_utime.labels(self.desc).inc(diff.ru_utime) + _background_process_ru_stime.labels(self.desc).inc(diff.ru_stime) + _background_process_db_txn_count.labels(self.desc).inc( + diff.db_txn_count, + ) + _background_process_db_txn_duration.labels(self.desc).inc( + diff.db_txn_duration_sec, + ) + _background_process_db_sched_duration.labels(self.desc).inc( + diff.db_sched_duration_sec, + ) + + +def run_as_background_process(desc, func, *args, **kwargs): + """Run the given function in its own logcontext, with resource metrics + + This should be used to wrap processes which are fired off to run in the + background, instead of being associated with a particular request. + + Args: + desc (str): a description for this background process type + func: a function, which may return a Deferred + args: positional args for func + kwargs: keyword args for func + + Returns: None + """ + @defer.inlineCallbacks + def run(): + count = _background_process_counts.get(desc, 0) + _background_process_counts[desc] = count + 1 + _background_process_start_count.labels(desc).inc() + + with LoggingContext(desc) as context: + context.request = "%s-%i" % (desc, count) + proc = _BackgroundProcess(desc, context) + _background_processes.setdefault(desc, set()).add(proc) + try: + yield func(*args, **kwargs) + finally: + proc.update_metrics() + _background_processes[desc].remove(proc) + + with PreserveLoggingContext(): + run() -- cgit 1.5.1 From 92aecd557bdd4c6eb60bdb7afa449ac6ba0c4a0f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 11:29:48 +0100 Subject: changelog --- changelog.d/3553.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3553.feature diff --git a/changelog.d/3553.feature b/changelog.d/3553.feature new file mode 100644 index 0000000000..77a294cb9f --- /dev/null +++ b/changelog.d/3553.feature @@ -0,0 +1 @@ +Add metrics to track resource usage by background processes -- cgit 1.5.1 From e45a46b6e4990befe6567aaf26354320dce4337a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Jul 2018 13:59:36 +0100 Subject: Add response code to response timer metrics --- synapse/http/request_metrics.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index f24b4b949c..c029a69ee5 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -38,7 +38,7 @@ outgoing_responses_counter = Counter( ) response_timer = Histogram( - "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag"] + "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag", "code"] ) response_ru_utime = Counter( @@ -171,11 +171,13 @@ class RequestMetrics(object): ) return - outgoing_responses_counter.labels(request.method, str(request.code)).inc() + response_code = str(request.code) + + outgoing_responses_counter.labels(request.method, response_code).inc() response_count.labels(request.method, self.name, tag).inc() - response_timer.labels(request.method, self.name, tag).observe( + response_timer.labels(request.method, self.name, tag, response_code).observe( time_sec - self.start ) -- cgit 1.5.1 From 00845c49d287a60100c73790225cc168801eb52f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Jul 2018 14:03:16 +0100 Subject: Newsfile --- changelog.d/3554.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3554.feature diff --git a/changelog.d/3554.feature b/changelog.d/3554.feature new file mode 100644 index 0000000000..b00397872c --- /dev/null +++ b/changelog.d/3554.feature @@ -0,0 +1 @@ +Add `code` label to `synapse_http_server_response_time_seconds` prometheus metric -- cgit 1.5.1 From 5bd0a47fcdea7dada06dc5d42e4340c8748386e8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Jul 2018 14:19:00 +0100 Subject: pep8 --- synapse/http/request_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index c029a69ee5..588e280571 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -38,7 +38,8 @@ outgoing_responses_counter = Counter( ) response_timer = Histogram( - "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag", "code"] + "synapse_http_server_response_time_seconds", "sec", + ["method", "servlet", "tag", "code"], ) response_ru_utime = Counter( -- cgit 1.5.1 From 3f9e649f1720b3419fd8008771955714baa9600b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 20:55:37 +0100 Subject: add config for pep8 Since, for better or worse, we seem to have configured isort to generate 89-character lines, pycharm is now complaining at me that our lines are too long. So, let's configure pep8 to behave consistently with flake8. --- setup.cfg | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9b5b75cd60..3f84283a38 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,12 +14,17 @@ ignore = pylint.cfg tox.ini -[flake8] +[pep8] max-line-length = 90 -# W503 requires that binary operators be at the end, not start, of lines. Erik doesn't like it. -# E203 is contrary to PEP8. +# W503 requires that binary operators be at the end, not start, of lines. Erik +# doesn't like it. E203 is contrary to PEP8. ignore = W503,E203 +[flake8] +# note that flake8 inherits the "ignore" settings from "pep8" (because it uses +# pep8 to do those checks), but not the "max-line-length" setting +max-line-length = 90 + [isort] line_length = 89 not_skip = __init__.py -- cgit 1.5.1 From eed24893fabb95b9ebbda58db54831f17dbdf2e6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 22:12:19 +0100 Subject: changelog --- changelog.d/3559.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3559.misc diff --git a/changelog.d/3559.misc b/changelog.d/3559.misc new file mode 100644 index 0000000000..26df859e45 --- /dev/null +++ b/changelog.d/3559.misc @@ -0,0 +1 @@ +add config for pep8 -- cgit 1.5.1