summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.editorconfig9
-rw-r--r--MANIFEST.in1
-rw-r--r--changelog.d/4189.misc2
-rw-r--r--changelog.d/4205.misc1
-rw-r--r--changelog.d/4241.bugfix1
-rw-r--r--changelog.d/4250.bugfix1
-rw-r--r--changelog.d/4257.misc1
-rw-r--r--synapse/appservice/scheduler.py36
-rw-r--r--synapse/rest/client/v1/pusher.py2
-rw-r--r--synapse/state/v1.py4
-rw-r--r--tests/__init__.py7
-rw-r--r--tests/patch_inline_callbacks.py90
-rw-r--r--tests/unittest.py4
13 files changed, 142 insertions, 17 deletions
diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000000..3edf9e717c
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,9 @@
+# EditorConfig https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4
diff --git a/MANIFEST.in b/MANIFEST.in
index d0e49713da..ec18819bc9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -26,6 +26,7 @@ recursive-include synapse/static *.js
 exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
+exclude .editorconfig
 
 include pyproject.toml
 recursive-include changelog.d *
diff --git a/changelog.d/4189.misc b/changelog.d/4189.misc
new file mode 100644
index 0000000000..4a41357d94
--- /dev/null
+++ b/changelog.d/4189.misc
@@ -0,0 +1,2 @@
+Run the AS senders as background processes to fix warnings
+
diff --git a/changelog.d/4205.misc b/changelog.d/4205.misc
new file mode 100644
index 0000000000..bbdce2c7aa
--- /dev/null
+++ b/changelog.d/4205.misc
@@ -0,0 +1 @@
+More logcontext checking in unittests
diff --git a/changelog.d/4241.bugfix b/changelog.d/4241.bugfix
new file mode 100644
index 0000000000..1158a5aa16
--- /dev/null
+++ b/changelog.d/4241.bugfix
@@ -0,0 +1 @@
+Fix exception caused by non-ascii event IDs
diff --git a/changelog.d/4250.bugfix b/changelog.d/4250.bugfix
new file mode 100644
index 0000000000..1f60f5bd0a
--- /dev/null
+++ b/changelog.d/4250.bugfix
@@ -0,0 +1 @@
+Pushers can now be unsubscribed from on Python 3.
diff --git a/changelog.d/4257.misc b/changelog.d/4257.misc
new file mode 100644
index 0000000000..43ac24cb7d
--- /dev/null
+++ b/changelog.d/4257.misc
@@ -0,0 +1 @@
+Add a basic .editorconfig
diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py
index 2430814796..685f15c061 100644
--- a/synapse/appservice/scheduler.py
+++ b/synapse/appservice/scheduler.py
@@ -53,8 +53,8 @@ import logging
 from twisted.internet import defer
 
 from synapse.appservice import ApplicationServiceState
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.logcontext import run_in_background
-from synapse.util.metrics import Measure
 
 logger = logging.getLogger(__name__)
 
@@ -104,27 +104,34 @@ class _ServiceQueuer(object):
         self.clock = clock
 
     def enqueue(self, service, event):
-        # if this service isn't being sent something
         self.queued_events.setdefault(service.id, []).append(event)
-        run_in_background(self._send_request, service)
 
-    @defer.inlineCallbacks
-    def _send_request(self, service):
+        # start a sender for this appservice if we don't already have one
+
         if service.id in self.requests_in_flight:
             return
 
+        run_as_background_process(
+            "as-sender-%s" % (service.id, ),
+            self._send_request, service,
+        )
+
+    @defer.inlineCallbacks
+    def _send_request(self, service):
+        # sanity-check: we shouldn't get here if this service already has a sender
+        # running.
+        assert(service.id not in self.requests_in_flight)
+
         self.requests_in_flight.add(service.id)
         try:
             while True:
                 events = self.queued_events.pop(service.id, [])
                 if not events:
                     return
-
-                with Measure(self.clock, "servicequeuer.send"):
-                    try:
-                        yield self.txn_ctrl.send(service, events)
-                    except Exception:
-                        logger.exception("AS request failed")
+                try:
+                    yield self.txn_ctrl.send(service, events)
+                except Exception:
+                    logger.exception("AS request failed")
         finally:
             self.requests_in_flight.discard(service.id)
 
@@ -223,7 +230,12 @@ class _Recoverer(object):
         self.backoff_counter = 1
 
     def recover(self):
-        self.clock.call_later((2 ** self.backoff_counter), self.retry)
+        def _retry():
+            run_as_background_process(
+                "as-recoverer-%s" % (self.service.id,),
+                self.retry,
+            )
+        self.clock.call_later((2 ** self.backoff_counter), _retry)
 
     def _backoff(self):
         # cap the backoff to be around 8.5min => (2^9) = 512 secs
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index b84f0260f2..4c07ae7f45 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -142,7 +142,7 @@ class PushersRemoveRestServlet(RestServlet):
     To allow pusher to be delete by clicking a link (ie. GET request)
     """
     PATTERNS = client_path_patterns("/pushers/remove$")
-    SUCCESS_HTML = "<html><body>You have been unsubscribed</body><html>"
+    SUCCESS_HTML = b"<html><body>You have been unsubscribed</body><html>"
 
     def __init__(self, hs):
         super(PushersRemoveRestServlet, self).__init__()
diff --git a/synapse/state/v1.py b/synapse/state/v1.py
index 70a981f4a2..19e091ce3b 100644
--- a/synapse/state/v1.py
+++ b/synapse/state/v1.py
@@ -298,6 +298,8 @@ def _resolve_normal_events(events, auth_events):
 
 def _ordered_events(events):
     def key_func(e):
-        return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest()
+        # we have to use utf-8 rather than ascii here because it turns out we allow
+        # people to send us events with non-ascii event IDs :/
+        return -int(e.depth), hashlib.sha1(e.event_id.encode('utf-8')).hexdigest()
 
     return sorted(events, key=key_func)
diff --git a/tests/__init__.py b/tests/__init__.py
index 9d9ca22829..d3181f9403 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +16,9 @@
 
 from twisted.trial import util
 
-from tests import utils
+import tests.patch_inline_callbacks
+
+# attempt to do the patch before we load any synapse code
+tests.patch_inline_callbacks.do_patch()
 
 util.DEFAULT_TIMEOUT_DURATION = 10
-utils.setupdb()
diff --git a/tests/patch_inline_callbacks.py b/tests/patch_inline_callbacks.py
new file mode 100644
index 0000000000..0f613945c8
--- /dev/null
+++ b/tests/patch_inline_callbacks.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import functools
+import sys
+
+from twisted.internet import defer
+from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
+
+
+def do_patch():
+    """
+    Patch defer.inlineCallbacks so that it checks the state of the logcontext on exit
+    """
+
+    from synapse.util.logcontext import LoggingContext
+
+    orig_inline_callbacks = defer.inlineCallbacks
+
+    def new_inline_callbacks(f):
+
+        orig = orig_inline_callbacks(f)
+
+        @functools.wraps(f)
+        def wrapped(*args, **kwargs):
+            start_context = LoggingContext.current_context()
+
+            try:
+                res = orig(*args, **kwargs)
+            except Exception:
+                if LoggingContext.current_context() != start_context:
+                    err = "%s changed context from %s to %s on exception" % (
+                        f, start_context, LoggingContext.current_context()
+                    )
+                    print(err, file=sys.stderr)
+                    raise Exception(err)
+                raise
+
+            if not isinstance(res, Deferred) or res.called:
+                if LoggingContext.current_context() != start_context:
+                    err = "%s changed context from %s to %s" % (
+                        f, start_context, LoggingContext.current_context()
+                    )
+                    # print the error to stderr because otherwise all we
+                    # see in travis-ci is the 500 error
+                    print(err, file=sys.stderr)
+                    raise Exception(err)
+                return res
+
+            if LoggingContext.current_context() != LoggingContext.sentinel:
+                err = (
+                    "%s returned incomplete deferred in non-sentinel context "
+                    "%s (start was %s)"
+                ) % (
+                    f, LoggingContext.current_context(), start_context,
+                )
+                print(err, file=sys.stderr)
+                raise Exception(err)
+
+            def check_ctx(r):
+                if LoggingContext.current_context() != start_context:
+                    err = "%s completion of %s changed context from %s to %s" % (
+                        "Failure" if isinstance(r, Failure) else "Success",
+                        f, start_context, LoggingContext.current_context(),
+                    )
+                    print(err, file=sys.stderr)
+                    raise Exception(err)
+                return r
+
+            res.addBoth(check_ctx)
+            return res
+
+        return wrapped
+
+    defer.inlineCallbacks = new_inline_callbacks
diff --git a/tests/unittest.py b/tests/unittest.py
index a191cccc29..092c930396 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -34,7 +34,9 @@ from synapse.types import UserID, create_requester
 from synapse.util.logcontext import LoggingContext, LoggingContextFilter
 
 from tests.server import get_clock, make_request, render, setup_test_homeserver
-from tests.utils import default_config
+from tests.utils import default_config, setupdb
+
+setupdb()
 
 # Set up putting Synapse's logs into Trial's.
 rootLogger = logging.getLogger()
generated by cgit-pink 1.4.1 (git 2.36.1) at 2024-11-11 05:08:53 +0000