Merge branch 'develop' into rav/cleanup_metrics

1 files changed, 199 insertions, 152 deletions

diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 099f9545ab..809199fe88 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -42,17 +42,21 @@ from synapse.visibility import filter_events_for_client
 logger = logging.getLogger(__name__)
 
 
-MESSAGE_FROM_PERSON_IN_ROOM = "You have a message on %(app)s from %(person)s " \
-                              "in the %(room)s room..."
+MESSAGE_FROM_PERSON_IN_ROOM = (
+    "You have a message on %(app)s from %(person)s " "in the %(room)s room..."
+)
 MESSAGE_FROM_PERSON = "You have a message on %(app)s from %(person)s..."
 MESSAGES_FROM_PERSON = "You have messages on %(app)s from %(person)s..."
 MESSAGES_IN_ROOM = "You have messages on %(app)s in the %(room)s room..."
-MESSAGES_IN_ROOM_AND_OTHERS = \
+MESSAGES_IN_ROOM_AND_OTHERS = (
     "You have messages on %(app)s in the %(room)s room and others..."
-MESSAGES_FROM_PERSON_AND_OTHERS = \
+)
+MESSAGES_FROM_PERSON_AND_OTHERS = (
     "You have messages on %(app)s from %(person)s and others..."
-INVITE_FROM_PERSON_TO_ROOM = "%(person)s has invited you to join the " \
-                             "%(room)s room on %(app)s..."
+)
+INVITE_FROM_PERSON_TO_ROOM = (
+    "%(person)s has invited you to join the " "%(room)s room on %(app)s..."
+)
 INVITE_FROM_PERSON = "%(person)s has invited you to chat on %(app)s..."
 
 CONTEXT_BEFORE = 1
@@ -60,12 +64,38 @@ CONTEXT_AFTER = 1
 
 # From https://github.com/matrix-org/matrix-react-sdk/blob/master/src/HtmlUtils.js
 ALLOWED_TAGS = [
-    'font',  # custom to matrix for IRC-style font coloring
-    'del',  # for markdown
+    "font",  # custom to matrix for IRC-style font coloring
+    "del",  # for markdown
     # deliberately no h1/h2 to stop people shouting.
-    'h3', 'h4', 'h5', 'h6', 'blockquote', 'p', 'a', 'ul', 'ol',
-    'nl', 'li', 'b', 'i', 'u', 'strong', 'em', 'strike', 'code', 'hr', 'br', 'div',
-    'table', 'thead', 'caption', 'tbody', 'tr', 'th', 'td', 'pre'
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "blockquote",
+    "p",
+    "a",
+    "ul",
+    "ol",
+    "nl",
+    "li",
+    "b",
+    "i",
+    "u",
+    "strong",
+    "em",
+    "strike",
+    "code",
+    "hr",
+    "br",
+    "div",
+    "table",
+    "thead",
+    "caption",
+    "tbody",
+    "tr",
+    "th",
+    "td",
+    "pre",
 ]
 ALLOWED_ATTRS = {
     # custom ones first:
@@ -94,13 +124,7 @@ class Mailer(object):
         logger.info("Created Mailer for app_name %s" % app_name)
 
     @defer.inlineCallbacks
-    def send_password_reset_mail(
-        self,
-        email_address,
-        token,
-        client_secret,
-        sid,
-    ):
+    def send_password_reset_mail(self, email_address, token, client_secret, sid):
         """Send an email with a password reset link to a user
 
         Args:
@@ -112,19 +136,16 @@ class Mailer(object):
                 group together multiple email sending attempts
             sid (str): The generated session ID
         """
-        if email.utils.parseaddr(email_address)[1] == '':
+        if email.utils.parseaddr(email_address)[1] == "":
             raise RuntimeError("Invalid 'to' email address")
 
         link = (
-            self.hs.config.public_baseurl +
-            "_matrix/client/unstable/password_reset/email/submit_token"
-            "?token=%s&client_secret=%s&sid=%s" %
-            (token, client_secret, sid)
+            self.hs.config.public_baseurl
+            + "_matrix/client/unstable/password_reset/email/submit_token"
+            "?token=%s&client_secret=%s&sid=%s" % (token, client_secret, sid)
         )
 
-        template_vars = {
-            "link": link,
-        }
+        template_vars = {"link": link}
 
         yield self.send_email(
             email_address,
@@ -133,15 +154,14 @@ class Mailer(object):
         )
 
     @defer.inlineCallbacks
-    def send_notification_mail(self, app_id, user_id, email_address,
-                               push_actions, reason):
+    def send_notification_mail(
+        self, app_id, user_id, email_address, push_actions, reason
+    ):
         """Send email regarding a user's room notifications"""
-        rooms_in_order = deduped_ordered_list(
-            [pa['room_id'] for pa in push_actions]
-        )
+        rooms_in_order = deduped_ordered_list([pa["room_id"] for pa in push_actions])
 
         notif_events = yield self.store.get_events(
-            [pa['event_id'] for pa in push_actions]
+            [pa["event_id"] for pa in push_actions]
         )
 
         notifs_by_room = {}
@@ -171,9 +191,7 @@ class Mailer(object):
         yield concurrently_execute(_fetch_room_state, rooms_in_order, 3)
 
         # actually sort our so-called rooms_in_order list, most recent room first
-        rooms_in_order.sort(
-            key=lambda r: -(notifs_by_room[r][-1]['received_ts'] or 0)
-        )
+        rooms_in_order.sort(key=lambda r: -(notifs_by_room[r][-1]["received_ts"] or 0))
 
         rooms = []
 
@@ -183,9 +201,11 @@ class Mailer(object):
             )
             rooms.append(roomvars)
 
-        reason['room_name'] = yield calculate_room_name(
-            self.store, state_by_room[reason['room_id']], user_id,
-            fallback_to_members=True
+        reason["room_name"] = yield calculate_room_name(
+            self.store,
+            state_by_room[reason["room_id"]],
+            user_id,
+            fallback_to_members=True,
         )
 
         summary_text = yield self.make_summary_text(
@@ -204,25 +224,21 @@ class Mailer(object):
         }
 
         yield self.send_email(
-            email_address,
-            "[%s] %s" % (self.app_name, summary_text),
-            template_vars,
+            email_address, "[%s] %s" % (self.app_name, summary_text), template_vars
         )
 
     @defer.inlineCallbacks
     def send_email(self, email_address, subject, template_vars):
         """Send an email with the given information and template text"""
         try:
-            from_string = self.hs.config.email_notif_from % {
-                "app": self.app_name
-            }
+            from_string = self.hs.config.email_notif_from % {"app": self.app_name}
         except TypeError:
             from_string = self.hs.config.email_notif_from
 
         raw_from = email.utils.parseaddr(from_string)[1]
         raw_to = email.utils.parseaddr(email_address)[1]
 
-        if raw_to == '':
+        if raw_to == "":
             raise RuntimeError("Invalid 'to' address")
 
         html_text = self.template_html.render(**template_vars)
@@ -231,27 +247,31 @@ class Mailer(object):
         plain_text = self.template_text.render(**template_vars)
         text_part = MIMEText(plain_text, "plain", "utf8")
 
-        multipart_msg = MIMEMultipart('alternative')
-        multipart_msg['Subject'] = subject
-        multipart_msg['From'] = from_string
-        multipart_msg['To'] = email_address
-        multipart_msg['Date'] = email.utils.formatdate()
-        multipart_msg['Message-ID'] = email.utils.make_msgid()
+        multipart_msg = MIMEMultipart("alternative")
+        multipart_msg["Subject"] = subject
+        multipart_msg["From"] = from_string
+        multipart_msg["To"] = email_address
+        multipart_msg["Date"] = email.utils.formatdate()
+        multipart_msg["Message-ID"] = email.utils.make_msgid()
         multipart_msg.attach(text_part)
         multipart_msg.attach(html_part)
 
-        logger.info("Sending email push notification to %s" % email_address)
-
-        yield make_deferred_yieldable(self.sendmail(
-            self.hs.config.email_smtp_host,
-            raw_from, raw_to, multipart_msg.as_string().encode('utf8'),
-            reactor=self.hs.get_reactor(),
-            port=self.hs.config.email_smtp_port,
-            requireAuthentication=self.hs.config.email_smtp_user is not None,
-            username=self.hs.config.email_smtp_user,
-            password=self.hs.config.email_smtp_pass,
-            requireTransportSecurity=self.hs.config.require_transport_security
-        ))
+        logger.info("Sending email notification to %s" % email_address)
+
+        yield make_deferred_yieldable(
+            self.sendmail(
+                self.hs.config.email_smtp_host,
+                raw_from,
+                raw_to,
+                multipart_msg.as_string().encode("utf8"),
+                reactor=self.hs.get_reactor(),
+                port=self.hs.config.email_smtp_port,
+                requireAuthentication=self.hs.config.email_smtp_user is not None,
+                username=self.hs.config.email_smtp_user,
+                password=self.hs.config.email_smtp_pass,
+                requireTransportSecurity=self.hs.config.require_transport_security,
+            )
+        )
 
     @defer.inlineCallbacks
     def get_room_vars(self, room_id, user_id, notifs, notif_events, room_state_ids):
@@ -272,17 +292,18 @@ class Mailer(object):
         if not is_invite:
             for n in notifs:
                 notifvars = yield self.get_notif_vars(
-                    n, user_id, notif_events[n['event_id']], room_state_ids
+                    n, user_id, notif_events[n["event_id"]], room_state_ids
                 )
 
                 # merge overlapping notifs together.
                 # relies on the notifs being in chronological order.
                 merge = False
-                if room_vars['notifs'] and 'messages' in room_vars['notifs'][-1]:
-                    prev_messages = room_vars['notifs'][-1]['messages']
-                    for message in notifvars['messages']:
-                        pm = list(filter(lambda pm: pm['id'] == message['id'],
-                                         prev_messages))
+                if room_vars["notifs"] and "messages" in room_vars["notifs"][-1]:
+                    prev_messages = room_vars["notifs"][-1]["messages"]
+                    for message in notifvars["messages"]:
+                        pm = list(
+                            filter(lambda pm: pm["id"] == message["id"], prev_messages)
+                        )
                         if pm:
                             if not message["is_historical"]:
                                 pm[0]["is_historical"] = False
@@ -293,20 +314,22 @@ class Mailer(object):
                             prev_messages.append(message)
 
                 if not merge:
-                    room_vars['notifs'].append(notifvars)
+                    room_vars["notifs"].append(notifvars)
 
         defer.returnValue(room_vars)
 
     @defer.inlineCallbacks
     def get_notif_vars(self, notif, user_id, notif_event, room_state_ids):
         results = yield self.store.get_events_around(
-            notif['room_id'], notif['event_id'],
-            before_limit=CONTEXT_BEFORE, after_limit=CONTEXT_AFTER
+            notif["room_id"],
+            notif["event_id"],
+            before_limit=CONTEXT_BEFORE,
+            after_limit=CONTEXT_AFTER,
         )
 
         ret = {
             "link": self.make_notif_link(notif),
-            "ts": notif['received_ts'],
+            "ts": notif["received_ts"],
             "messages": [],
         }
 
@@ -318,7 +341,7 @@ class Mailer(object):
         for event in the_events:
             messagevars = yield self.get_message_vars(notif, event, room_state_ids)
             if messagevars is not None:
-                ret['messages'].append(messagevars)
+                ret["messages"].append(messagevars)
 
         defer.returnValue(ret)
 
@@ -340,7 +363,7 @@ class Mailer(object):
 
         ret = {
             "msgtype": msgtype,
-            "is_historical": event.event_id != notif['event_id'],
+            "is_historical": event.event_id != notif["event_id"],
             "id": event.event_id,
             "ts": event.origin_server_ts,
             "sender_name": sender_name,
@@ -379,8 +402,9 @@ class Mailer(object):
         return messagevars
 
     @defer.inlineCallbacks
-    def make_summary_text(self, notifs_by_room, room_state_ids,
-                          notif_events, user_id, reason):
+    def make_summary_text(
+        self, notifs_by_room, room_state_ids, notif_events, user_id, reason
+    ):
         if len(notifs_by_room) == 1:
             # Only one room has new stuff
             room_id = list(notifs_by_room.keys())[0]
@@ -404,16 +428,19 @@ class Mailer(object):
                 inviter_name = name_from_member_event(inviter_member_event)
 
                 if room_name is None:
-                    defer.returnValue(INVITE_FROM_PERSON % {
-                        "person": inviter_name,
-                        "app": self.app_name
-                    })
+                    defer.returnValue(
+                        INVITE_FROM_PERSON
+                        % {"person": inviter_name, "app": self.app_name}
+                    )
                 else:
-                    defer.returnValue(INVITE_FROM_PERSON_TO_ROOM % {
-                        "person": inviter_name,
-                        "room": room_name,
-                        "app": self.app_name,
-                    })
+                    defer.returnValue(
+                        INVITE_FROM_PERSON_TO_ROOM
+                        % {
+                            "person": inviter_name,
+                            "room": room_name,
+                            "app": self.app_name,
+                        }
+                    )
 
             sender_name = None
             if len(notifs_by_room[room_id]) == 1:
@@ -427,67 +454,86 @@ class Mailer(object):
                     sender_name = name_from_member_event(state_event)
 
                 if sender_name is not None and room_name is not None:
-                    defer.returnValue(MESSAGE_FROM_PERSON_IN_ROOM % {
-                        "person": sender_name,
-                        "room": room_name,
-                        "app": self.app_name,
-                    })
+                    defer.returnValue(
+                        MESSAGE_FROM_PERSON_IN_ROOM
+                        % {
+                            "person": sender_name,
+                            "room": room_name,
+                            "app": self.app_name,
+                        }
+                    )
                 elif sender_name is not None:
-                    defer.returnValue(MESSAGE_FROM_PERSON % {
-                        "person": sender_name,
-                        "app": self.app_name,
-                    })
+                    defer.returnValue(
+                        MESSAGE_FROM_PERSON
+                        % {"person": sender_name, "app": self.app_name}
+                    )
             else:
                 # There's more than one notification for this room, so just
                 # say there are several
                 if room_name is not None:
-                    defer.returnValue(MESSAGES_IN_ROOM % {
-                        "room": room_name,
-                        "app": self.app_name,
-                    })
+                    defer.returnValue(
+                        MESSAGES_IN_ROOM % {"room": room_name, "app": self.app_name}
+                    )
                 else:
                     # If the room doesn't have a name, say who the messages
                     # are from explicitly to avoid, "messages in the Bob room"
-                    sender_ids = list(set([
-                        notif_events[n['event_id']].sender
-                        for n in notifs_by_room[room_id]
-                    ]))
-
-                    member_events = yield self.store.get_events([
-                        room_state_ids[room_id][("m.room.member", s)]
-                        for s in sender_ids
-                    ])
-
-                    defer.returnValue(MESSAGES_FROM_PERSON % {
-                        "person": descriptor_from_member_events(member_events.values()),
-                        "app": self.app_name,
-                    })
+                    sender_ids = list(
+                        set(
+                            [
+                                notif_events[n["event_id"]].sender
+                                for n in notifs_by_room[room_id]
+                            ]
+                        )
+                    )
+
+                    member_events = yield self.store.get_events(
+                        [
+                            room_state_ids[room_id][("m.room.member", s)]
+                            for s in sender_ids
+                        ]
+                    )
+
+                    defer.returnValue(
+                        MESSAGES_FROM_PERSON
+                        % {
+                            "person": descriptor_from_member_events(
+                                member_events.values()
+                            ),
+                            "app": self.app_name,
+                        }
+                    )
         else:
             # Stuff's happened in multiple different rooms
 
             # ...but we still refer to the 'reason' room which triggered the mail
-            if reason['room_name'] is not None:
-                defer.returnValue(MESSAGES_IN_ROOM_AND_OTHERS % {
-                    "room": reason['room_name'],
-                    "app": self.app_name,
-                })
+            if reason["room_name"] is not None:
+                defer.returnValue(
+                    MESSAGES_IN_ROOM_AND_OTHERS
+                    % {"room": reason["room_name"], "app": self.app_name}
+                )
             else:
                 # If the reason room doesn't have a name, say who the messages
                 # are from explicitly to avoid, "messages in the Bob room"
-                sender_ids = list(set([
-                    notif_events[n['event_id']].sender
-                    for n in notifs_by_room[reason['room_id']]
-                ]))
+                sender_ids = list(
+                    set(
+                        [
+                            notif_events[n["event_id"]].sender
+                            for n in notifs_by_room[reason["room_id"]]
+                        ]
+                    )
+                )
 
-                member_events = yield self.store.get_events([
-                    room_state_ids[room_id][("m.room.member", s)]
-                    for s in sender_ids
-                ])
+                member_events = yield self.store.get_events(
+                    [room_state_ids[room_id][("m.room.member", s)] for s in sender_ids]
+                )
 
-                defer.returnValue(MESSAGES_FROM_PERSON_AND_OTHERS % {
-                    "person": descriptor_from_member_events(member_events.values()),
-                    "app": self.app_name,
-                })
+                defer.returnValue(
+                    MESSAGES_FROM_PERSON_AND_OTHERS
+                    % {
+                        "person": descriptor_from_member_events(member_events.values()),
+                        "app": self.app_name,
+                    }
+                )
 
     def make_room_link(self, room_id):
         if self.hs.config.email_riot_base_url:
@@ -503,17 +549,17 @@ class Mailer(object):
         if self.hs.config.email_riot_base_url:
             return "%s/#/room/%s/%s" % (
                 self.hs.config.email_riot_base_url,
-                notif['room_id'], notif['event_id']
+                notif["room_id"],
+                notif["event_id"],
             )
         elif self.app_name == "Vector":
             # need /beta for Universal Links to work on iOS
             return "https://vector.im/beta/#/room/%s/%s" % (
-                notif['room_id'], notif['event_id']
+                notif["room_id"],
+                notif["event_id"],
             )
         else:
-            return "https://matrix.to/#/%s/%s" % (
-                notif['room_id'], notif['event_id']
-            )
+            return "https://matrix.to/#/%s/%s" % (notif["room_id"], notif["event_id"])
 
     def make_unsubscribe_link(self, user_id, app_id, email_address):
         params = {
@@ -530,12 +576,18 @@ class Mailer(object):
 
 
 def safe_markup(raw_html):
-    return jinja2.Markup(bleach.linkify(bleach.clean(
-        raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRS,
-        # bleach master has this, but it isn't released yet
-        # protocols=ALLOWED_SCHEMES,
-        strip=True
-    )))
+    return jinja2.Markup(
+        bleach.linkify(
+            bleach.clean(
+                raw_html,
+                tags=ALLOWED_TAGS,
+                attributes=ALLOWED_ATTRS,
+                # bleach master has this, but it isn't released yet
+                # protocols=ALLOWED_SCHEMES,
+                strip=True,
+            )
+        )
+    )
 
 
 def safe_text(raw_text):
@@ -543,10 +595,9 @@ def safe_text(raw_text):
     Process text: treat it as HTML but escape any tags (ie. just escape the
     HTML) then linkify it.
     """
-    return jinja2.Markup(bleach.linkify(bleach.clean(
-        raw_text, tags=[], attributes={},
-        strip=False
-    )))
+    return jinja2.Markup(
+        bleach.linkify(bleach.clean(raw_text, tags=[], attributes={}, strip=False))
+    )
 
 
 def deduped_ordered_list(l):
@@ -595,15 +646,11 @@ def _create_mxc_to_http_filter(config):
 
         serverAndMediaId = value[6:]
         fragment = None
-        if '#' in serverAndMediaId:
-            (serverAndMediaId, fragment) = serverAndMediaId.split('#', 1)
+        if "#" in serverAndMediaId:
+            (serverAndMediaId, fragment) = serverAndMediaId.split("#", 1)
             fragment = "#" + fragment
 
-        params = {
-            "width": width,
-            "height": height,
-            "method": resize_method,
-        }
+        params = {"width": width, "height": height, "method": resize_method}
         return "%s_matrix/media/v1/thumbnail/%s?%s%s" % (
             config.public_baseurl,
             serverAndMediaId,