summary refs log tree commit diff
path: root/synapse/storage/databases/main/monthly_active_users.py
blob: 9a63f953fb5e739d5f7f4fbd39f957fd2d3cd139 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
# Copyright 2018 New Vector
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast

from synapse.metrics.background_process_metrics import wrap_as_background_process
from synapse.storage.database import (
    DatabasePool,
    LoggingDatabaseConnection,
    LoggingTransaction,
    make_in_list_sql_clause,
)
from synapse.storage.databases.main.registration import RegistrationWorkerStore
from synapse.util.caches.descriptors import cached
from synapse.util.threepids import canonicalise_email

if TYPE_CHECKING:
    from synapse.server import HomeServer

logger = logging.getLogger(__name__)

# Number of msec of granularity to store the monthly_active_user timestamp
# This means it is not necessary to update the table on every request
LAST_SEEN_GRANULARITY = 60 * 60 * 1000


class MonthlyActiveUsersWorkerStore(RegistrationWorkerStore):
    def __init__(
        self,
        database: DatabasePool,
        db_conn: LoggingDatabaseConnection,
        hs: "HomeServer",
    ):
        super().__init__(database, db_conn, hs)
        self._clock = hs.get_clock()
        self.hs = hs

        if hs.config.redis.redis_enabled:
            # If we're using Redis, we can shift this update process off to
            # the background worker
            self._update_on_this_worker = hs.config.worker.run_background_tasks
        else:
            # If we're NOT using Redis, this must be handled by the master
            self._update_on_this_worker = hs.get_instance_name() == "master"

        self._limit_usage_by_mau = hs.config.server.limit_usage_by_mau
        self._max_mau_value = hs.config.server.max_mau_value

        self._mau_stats_only = hs.config.server.mau_stats_only

        if self._update_on_this_worker:
            # Do not add more reserved users than the total allowable number
            self.db_pool.new_transaction(
                db_conn,
                "initialise_mau_threepids",
                [],
                [],
                self._initialise_reserved_users,
                hs.config.server.mau_limits_reserved_threepids[: self._max_mau_value],
            )

    @cached(num_args=0)
    async def get_monthly_active_count(self) -> int:
        """Generates current count of monthly active users

        Returns:
            Number of current monthly active users
        """

        def _count_users(txn: LoggingTransaction) -> int:
            # Exclude app service users
            sql = """
                SELECT COUNT(*)
                FROM monthly_active_users
                    LEFT JOIN users
                    ON monthly_active_users.user_id=users.name
                WHERE (users.appservice_id IS NULL OR users.appservice_id = '');
            """
            txn.execute(sql)
            (count,) = cast(Tuple[int], txn.fetchone())
            return count

        return await self.db_pool.runInteraction("count_users", _count_users)

    @cached(num_args=0)
    async def get_monthly_active_count_by_service(self) -> Dict[str, int]:
        """Generates current count of monthly active users broken down by service.
        A service is typically an appservice but also includes native matrix users.
        Since the `monthly_active_users` table is populated from the `user_ips` table
        `config.appservice.track_appservice_user_ips` must be set to `true` for this
        method to return anything other than native matrix users.

        Returns:
            A mapping between app_service_id and the number of occurrences.

        """

        def _count_users_by_service(txn: LoggingTransaction) -> Dict[str, int]:
            sql = """
                SELECT COALESCE(appservice_id, 'native'), COUNT(*)
                FROM monthly_active_users
                LEFT JOIN users ON monthly_active_users.user_id=users.name
                GROUP BY appservice_id;
            """

            txn.execute(sql)
            result = cast(List[Tuple[str, int]], txn.fetchall())
            return dict(result)

        return await self.db_pool.runInteraction(
            "count_users_by_service", _count_users_by_service
        )

    async def get_monthly_active_users_by_service(
        self, start_timestamp: Optional[int] = None, end_timestamp: Optional[int] = None
    ) -> List[Tuple[str, str]]:
        """Generates list of monthly active users and their services.
        Please see "get_monthly_active_count_by_service" docstring for more details
        about services.

        Arguments:
            start_timestamp: If specified, only include users that were first active
                at or after this point
            end_timestamp: If specified, only include users that were first active
                at or before this point

        Returns:
            A list of tuples (appservice_id, user_id). "native" is emitted as the
            appservice for users that don't come from appservices (i.e. native Matrix
            users).

        """
        if start_timestamp is not None and end_timestamp is not None:
            where_clause = 'WHERE "timestamp" >= ? and "timestamp" <= ?'
            query_params = [start_timestamp, end_timestamp]
        elif start_timestamp is not None:
            where_clause = 'WHERE "timestamp" >= ?'
            query_params = [start_timestamp]
        elif end_timestamp is not None:
            where_clause = 'WHERE "timestamp" <= ?'
            query_params = [end_timestamp]
        else:
            where_clause = ""
            query_params = []

        def _list_users(txn: LoggingTransaction) -> List[Tuple[str, str]]:
            sql = f"""
                    SELECT COALESCE(appservice_id, 'native'), user_id
                    FROM monthly_active_users
                    LEFT JOIN users ON monthly_active_users.user_id=users.name
                    {where_clause};
                """

            txn.execute(sql, query_params)
            return cast(List[Tuple[str, str]], txn.fetchall())

        return await self.db_pool.runInteraction("list_users", _list_users)

    async def get_registered_reserved_users(self) -> List[str]:
        """Of the reserved threepids defined in config, retrieve those that are associated
        with registered users

        Returns:
            User IDs of actual users that are reserved
        """
        users = []

        for tp in self.hs.config.server.mau_limits_reserved_threepids[
            : self.hs.config.server.max_mau_value
        ]:
            user_id = await self.hs.get_datastores().main.get_user_id_by_threepid(
                tp["medium"], canonicalise_email(tp["address"])
            )
            if user_id:
                users.append(user_id)

        return users

    @cached(num_args=1)
    async def user_last_seen_monthly_active(self, user_id: str) -> Optional[int]:
        """
        Checks if a given user is part of the monthly active user group

        Arguments:
            user_id: user to add/update

        Return:
            Timestamp since last seen, None if never seen
        """

        return await self.db_pool.simple_select_one_onecol(
            table="monthly_active_users",
            keyvalues={"user_id": user_id},
            retcol="timestamp",
            allow_none=True,
            desc="user_last_seen_monthly_active",
        )

    @wrap_as_background_process("reap_monthly_active_users")
    async def reap_monthly_active_users(self) -> None:
        """Cleans out monthly active user table to ensure that no stale
        entries exist.
        """

        def _reap_users(txn: LoggingTransaction, reserved_users: List[str]) -> None:
            """
            Args:
                reserved_users (tuple): reserved users to preserve
            """

            thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)

            in_clause, in_clause_args = make_in_list_sql_clause(
                self.database_engine, "user_id", reserved_users
            )

            txn.execute(
                "DELETE FROM monthly_active_users WHERE timestamp < ? AND NOT %s"
                % (in_clause,),
                [thirty_days_ago] + in_clause_args,
            )

            if self._limit_usage_by_mau:
                # If MAU user count still exceeds the MAU threshold, then delete on
                # a least recently active basis.
                # Note it is not possible to write this query using OFFSET due to
                # incompatibilities in how sqlite and postgres support the feature.
                # Sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present,
                # while Postgres does not require 'LIMIT', but also does not support
                # negative LIMIT values. So there is no way to write it that both can
                # support

                # Limit must be >= 0 for postgres
                num_of_non_reserved_users_to_remove = max(
                    self._max_mau_value - len(reserved_users), 0
                )

                # It is important to filter reserved users twice to guard
                # against the case where the reserved user is present in the
                # SELECT, meaning that a legitimate mau is deleted.
                sql = """
                    DELETE FROM monthly_active_users
                    WHERE user_id NOT IN (
                        SELECT user_id FROM monthly_active_users
                        WHERE NOT %s
                        ORDER BY timestamp DESC
                        LIMIT ?
                    )
                    AND NOT %s
                """ % (
                    in_clause,
                    in_clause,
                )

                query_args = (
                    in_clause_args
                    + [num_of_non_reserved_users_to_remove]
                    + in_clause_args
                )
                txn.execute(sql, query_args)

            # It seems poor to invalidate the whole cache. Postgres supports
            # 'Returning' which would allow me to invalidate only the
            # specific users, but sqlite has no way to do this and instead
            # I would need to SELECT and the DELETE which without locking
            # is racy.
            # Have resolved to invalidate the whole cache for now and do
            # something about it if and when the perf becomes significant
            self._invalidate_all_cache_and_stream(
                txn, self.user_last_seen_monthly_active
            )
            self._invalidate_cache_and_stream(txn, self.get_monthly_active_count, ())

        reserved_users = await self.get_registered_reserved_users()
        await self.db_pool.runInteraction(
            "reap_monthly_active_users", _reap_users, reserved_users
        )

    def _initialise_reserved_users(
        self, txn: LoggingTransaction, threepids: List[dict]
    ) -> None:
        """Ensures that reserved threepids are accounted for in the MAU table, should
        be called on start up.

        Args:
            txn:
            threepids: List of threepid dicts to reserve
        """
        assert (
            self._update_on_this_worker
        ), "This worker is not designated to update MAUs"

        # XXX what is this function trying to achieve?  It upserts into
        # monthly_active_users for each *registered* reserved mau user, but why?
        #
        #  - shouldn't there already be an entry for each reserved user (at least
        #    if they have been active recently)?
        #
        #  - if it's important that the timestamp is kept up to date, why do we only
        #    run this at startup?

        for tp in threepids:
            user_id = self.get_user_id_by_threepid_txn(txn, tp["medium"], tp["address"])

            if user_id:
                is_support = self.is_support_user_txn(txn, user_id)
                if not is_support:
                    # We do this manually here to avoid hitting #6791
                    self.db_pool.simple_upsert_txn(
                        txn,
                        table="monthly_active_users",
                        keyvalues={"user_id": user_id},
                        values={"timestamp": int(self._clock.time_msec())},
                    )
            else:
                logger.warning("mau limit reserved threepid %s not found in db" % tp)

    async def upsert_monthly_active_user(self, user_id: str) -> None:
        """Updates or inserts the user into the monthly active user table, which
        is used to track the current MAU usage of the server

        Args:
            user_id: user to add/update
        """
        assert (
            self._update_on_this_worker
        ), "This worker is not designated to update MAUs"

        # Support user never to be included in MAU stats. Note I can't easily call this
        # from upsert_monthly_active_user_txn because then I need a _txn form of
        # is_support_user which is complicated because I want to cache the result.
        # Therefore I call it here and ignore the case where
        # upsert_monthly_active_user_txn is called directly from
        # _initialise_reserved_users reasoning that it would be very strange to
        #  include a support user in this context.

        is_support = await self.is_support_user(user_id)
        if is_support:
            return

        await self.db_pool.runInteraction(
            "upsert_monthly_active_user", self.upsert_monthly_active_user_txn, user_id
        )

    def upsert_monthly_active_user_txn(
        self, txn: LoggingTransaction, user_id: str
    ) -> None:
        """Updates or inserts monthly active user member

        We consciously do not call is_support_txn from this method because it
        is not possible to cache the response. is_support_txn will be false in
        almost all cases, so it seems reasonable to call it only for
        upsert_monthly_active_user and to call is_support_txn manually
        for cases where upsert_monthly_active_user_txn is called directly,
        like _initialise_reserved_users

        In short, don't call this method with support users. (Support users
        should not appear in the MAU stats).

        Args:
            txn (cursor):
            user_id (str): user to add/update
        """
        assert (
            self._update_on_this_worker
        ), "This worker is not designated to update MAUs"

        # Am consciously deciding to lock the table on the basis that is ought
        # never be a big table and alternative approaches (batching multiple
        # upserts into a single txn) introduced a lot of extra complexity.
        # See https://github.com/matrix-org/synapse/issues/3854 for more
        self.db_pool.simple_upsert_txn(
            txn,
            table="monthly_active_users",
            keyvalues={"user_id": user_id},
            values={"timestamp": int(self._clock.time_msec())},
        )

        self._invalidate_cache_and_stream(txn, self.get_monthly_active_count, ())
        self._invalidate_cache_and_stream(
            txn, self.get_monthly_active_count_by_service, ()
        )
        self._invalidate_cache_and_stream(
            txn, self.user_last_seen_monthly_active, (user_id,)
        )

    async def populate_monthly_active_users(self, user_id: str) -> None:
        """Checks on the state of monthly active user limits and optionally
        add the user to the monthly active tables

        Args:
            user_id(str): the user_id to query
        """
        assert (
            self._update_on_this_worker
        ), "This worker is not designated to update MAUs"

        if self._limit_usage_by_mau or self._mau_stats_only:
            # Trial users and guests should not be included as part of MAU group
            is_guest = await self.is_guest(user_id)
            if is_guest:
                return
            is_trial = await self.is_trial_user(user_id)
            if is_trial:
                return

            last_seen_timestamp = await self.user_last_seen_monthly_active(user_id)
            now = self.hs.get_clock().time_msec()

            # We want to reduce to the total number of db writes, and are happy
            # to trade accuracy of timestamp in order to lighten load. This means
            # We always insert new users (where MAU threshold has not been reached),
            # but only update if we have not previously seen the user for
            # LAST_SEEN_GRANULARITY ms
            if last_seen_timestamp is None:
                # In the case where mau_stats_only is True and limit_usage_by_mau is
                # False, there is no point in checking get_monthly_active_count - it
                # adds no value and will break the logic if max_mau_value is exceeded.
                if not self._limit_usage_by_mau:
                    await self.upsert_monthly_active_user(user_id)
                else:
                    count = await self.get_monthly_active_count()
                    if count < self._max_mau_value:
                        await self.upsert_monthly_active_user(user_id)
            elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
                await self.upsert_monthly_active_user(user_id)