From b84d59c5f01914fe53d2673c5c7e372f5c61d088 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 16:22:35 +0000 Subject: Add descriptions --- synapse/storage/registration.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse/storage/registration.py') diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 70cde0d04d..bd35e19be6 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -134,6 +134,7 @@ class RegistrationStore(SQLBaseStore): }, retcols=["name", "password_hash", "is_guest"], allow_none=True, + desc="get_user_by_id", ) def get_users_by_id_case_insensitive(self, user_id): -- cgit 1.5.1 From 79a1c0574b33955d28bfb12697ccd5a7be779b36 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 5 Feb 2016 11:22:30 +0000 Subject: Allocate guest user IDs numericcally The current random IDs are ugly and confusing when presented in UIs. This makes them prettier and easier to read. Also, disable non-automated registration of numeric IDs so that we don't need to worry so much about people carving out our automated address space and us needing to keep retrying ID registration. --- synapse/handlers/register.py | 55 +++++++++++++++++++++++++++-------------- synapse/storage/registration.py | 36 +++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 19 deletions(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b8fbcf9233..2660fd21a2 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -21,7 +21,6 @@ from synapse.api.errors import ( AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError ) from ._base import BaseHandler -import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor from synapse.http.client import CaptchaServerHttpClient @@ -45,6 +44,8 @@ class RegistrationHandler(BaseHandler): self.distributor.declare("registered_user") self.captcha_client = CaptchaServerHttpClient(hs) + self._next_generated_user_id = None + @defer.inlineCallbacks def check_username(self, localpart, guest_access_token=None): yield run_on_reactor() @@ -91,7 +92,7 @@ class RegistrationHandler(BaseHandler): Args: localpart : The local part of the user ID to register. If None, - one will be randomly generated. + one will be generated. password (str) : The password to assign to this user so they can login again. This can be None which means they cannot login again via a password (e.g. the user is an application service user). @@ -108,6 +109,18 @@ class RegistrationHandler(BaseHandler): if localpart: yield self.check_username(localpart, guest_access_token=guest_access_token) + was_guest = guest_access_token is not None + + if not was_guest: + try: + int(localpart) + raise RegistrationError( + 400, + "Numeric user IDs are reserved for guest users." + ) + except ValueError: + pass + user = UserID(localpart, self.hs.hostname) user_id = user.to_string() @@ -118,40 +131,36 @@ class RegistrationHandler(BaseHandler): user_id=user_id, token=token, password_hash=password_hash, - was_guest=guest_access_token is not None, + was_guest=was_guest, make_guest=make_guest, ) yield registered_user(self.distributor, user) else: - # autogen a random user ID + # autogen a sequential user ID attempts = 0 - user_id = None token = None - while not user_id: + user = None + while not user: + localpart = yield self._generate_user_id(attempts > 0) + user = UserID(localpart, self.hs.hostname) + user_id = user.to_string() + yield self.check_user_id_is_valid(user_id) + if generate_token: + token = self.auth_handler().generate_access_token(user_id) try: - localpart = self._generate_user_id() - user = UserID(localpart, self.hs.hostname) - user_id = user.to_string() - yield self.check_user_id_is_valid(user_id) - if generate_token: - token = self.auth_handler().generate_access_token(user_id) yield self.store.register( user_id=user_id, token=token, password_hash=password_hash, make_guest=make_guest ) - - yield registered_user(self.distributor, user) except SynapseError: # if user id is taken, just generate another user_id = None token = None attempts += 1 - if attempts > 5: - raise RegistrationError( - 500, "Cannot generate user ID.") + yield registered_user(self.distributor, user) # We used to generate default identicons here, but nowadays # we want clients to generate their own as part of their branding @@ -283,8 +292,16 @@ class RegistrationHandler(BaseHandler): errcode=Codes.EXCLUSIVE ) - def _generate_user_id(self): - return "-" + stringutils.random_string(18) + @defer.inlineCallbacks + def _generate_user_id(self, reseed=False): + if reseed or self._next_generated_user_id is None: + self._next_generated_user_id = ( + yield self.store.find_next_generated_user_id_localpart() + ) + + id = self._next_generated_user_id + self._next_generated_user_id += 1 + defer.returnValue(str(id)) @defer.inlineCallbacks def _validate_captcha(self, ip_addr, private_key, challenge, response): diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index bd35e19be6..967c732bda 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + from twisted.internet import defer from synapse.api.errors import StoreError, Codes @@ -351,3 +353,37 @@ class RegistrationStore(SQLBaseStore): ret = yield self.runInteraction("count_users", _count_users) defer.returnValue(ret) + + @defer.inlineCallbacks + def find_next_generated_user_id_localpart(self): + """ + Gets the localpart of the next generated user ID. + + Generated user IDs are integers, and we aim for them to be as small as + we can. Unfortunately, it's possible some of them are already taken by + existing users, and there may be gaps in the already taken range. This + function returns the start of the first allocatable gap. This is to + avoid the case of ID 10000000 being pre-allocated, so us wasting the + first (and shortest) many generated user IDs. + """ + def _find_next_generated_user_id(txn): + txn.execute("SELECT name FROM users") + rows = self.cursor_to_dict(txn) + + regex = re.compile("^@(\d+):") + + found = set() + + for r in rows: + user_id = r["name"] + match = regex.search(user_id) + if match: + found.add(int(match.group(1))) + for i in xrange(len(found) + 1): + if i not in found: + return i + + defer.returnValue((yield self.runInteraction( + "find_next_generated_user_id", + _find_next_generated_user_id + ))) -- cgit 1.5.1 From 33300673b7a6f79802f691ac121e720cb44c0dfc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 24 Feb 2016 14:41:25 +0000 Subject: Generate guest access token on 3pid invites This means that following the same link across multiple sessions or devices can re-use the same guest account. Note that this is somewhat of an abuse vector; we can't throw up captchas on this flow, so this is a way of registering ephemeral accounts for spam, whose sign-up we don't rate limit. --- synapse/handlers/register.py | 15 ++++++++ synapse/handlers/room.py | 8 ++++ synapse/storage/registration.py | 44 ++++++++++++++++++++++ .../delta/30/threepid_guest_access_tokens.sql | 24 ++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 synapse/storage/schema/delta/30/threepid_guest_access_tokens.sql (limited to 'synapse/storage/registration.py') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index f8959e5d82..6d155d57e7 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -349,3 +349,18 @@ class RegistrationHandler(BaseHandler): def auth_handler(self): return self.hs.get_handlers().auth_handler + + @defer.inlineCallbacks + def guest_access_token_for(self, medium, address, inviter_user_id): + access_token = yield self.store.get_3pid_guest_access_token(medium, address) + if access_token: + defer.returnValue(access_token) + + _, access_token = yield self.register( + generate_token=True, + make_guest=True + ) + access_token = yield self.store.save_or_get_3pid_guest_access_token( + medium, address, access_token, inviter_user_id + ) + defer.returnValue(access_token) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index eb9700a35b..d2de23a6cc 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -848,6 +848,13 @@ class RoomMemberHandler(BaseHandler): user. """ + registration_handler = self.hs.get_handlers().registration_handler + guest_access_token = yield registration_handler.guest_access_token_for( + medium=medium, + address=address, + inviter_user_id=inviter_user_id, + ) + is_url = "%s%s/_matrix/identity/api/v1/store-invite" % ( id_server_scheme, id_server, ) @@ -864,6 +871,7 @@ class RoomMemberHandler(BaseHandler): "sender": inviter_user_id, "sender_display_name": inviter_display_name, "sender_avatar_url": inviter_avatar_url, + "guest_access_token": guest_access_token, } ) # TODO: Check for success diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 967c732bda..03a9b66e4a 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -387,3 +387,47 @@ class RegistrationStore(SQLBaseStore): "find_next_generated_user_id", _find_next_generated_user_id ))) + + @defer.inlineCallbacks + def get_3pid_guest_access_token(self, medium, address): + ret = yield self._simple_select_one( + "threepid_guest_access_tokens", + { + "medium": medium, + "address": address + }, + ["guest_access_token"], True, 'get_3pid_guest_access_token' + ) + if ret: + defer.returnValue(ret["guest_access_token"]) + defer.returnValue(None) + + @defer.inlineCallbacks + def save_or_get_3pid_guest_access_token( + self, medium, address, access_token, inviter_user_id + ): + """ + Gets the 3pid's guest access token if exists, else saves access_token. + + :param medium (str): Medium of the 3pid. Must be "email". + :param address (str): 3pid address. + :param access_token (str): The access token to persist if none is + already persisted. + :param inviter_user_id (str): User ID of the inviter. + :return (deferred str): Whichever access token is persisted at the end + of this function call. + """ + def insert(txn): + txn.execute( + "INSERT INTO threepid_guest_access_tokens " + "(medium, address, guest_access_token, first_inviter) " + "VALUES (?, ?, ?, ?)", + (medium, address, access_token, inviter_user_id) + ) + + try: + yield self.runInteraction("save_3pid_guest_access_token", insert) + defer.returnValue(access_token) + except self.database_engine.module.IntegrityError: + ret = yield self.get_3pid_guest_access_token(medium, address) + defer.returnValue(ret) diff --git a/synapse/storage/schema/delta/30/threepid_guest_access_tokens.sql b/synapse/storage/schema/delta/30/threepid_guest_access_tokens.sql new file mode 100644 index 0000000000..0dd2f1360c --- /dev/null +++ b/synapse/storage/schema/delta/30/threepid_guest_access_tokens.sql @@ -0,0 +1,24 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Stores guest account access tokens generated for unbound 3pids. +CREATE TABLE threepid_guest_access_tokens( + medium TEXT, -- The medium of the 3pid. Must be "email". + address TEXT, -- The 3pid address. + guest_access_token TEXT, -- The access token for a guest user for this 3pid. + first_inviter TEXT -- User ID of the first user to invite this 3pid to a room. +); + +CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens(medium, address); -- cgit 1.5.1 From 54172924c834a954fcbbc6224318140b9e95aa7d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 1 Mar 2016 14:32:56 +0000 Subject: Load the current id in the IdGenerator constructor Rather than loading them lazily. This allows us to remove all the yield statements and spurious arguments for the get_next methods. It also allows us to replace all instances of get_next_txn with get_next since get_next no longer needs to access the db. --- synapse/storage/__init__.py | 14 +++---- synapse/storage/account_data.py | 8 ++-- synapse/storage/events.py | 6 +-- synapse/storage/presence.py | 4 +- synapse/storage/push_rule.py | 4 +- synapse/storage/pusher.py | 2 +- synapse/storage/receipts.py | 4 +- synapse/storage/registration.py | 6 +-- synapse/storage/state.py | 2 +- synapse/storage/tags.py | 8 ++-- synapse/storage/transactions.py | 2 +- synapse/storage/util/id_generators.py | 69 +++++++++++------------------------ 12 files changed, 52 insertions(+), 77 deletions(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9be1d12fac..f257721ea3 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -115,13 +115,13 @@ class DataStore(RoomMemberStore, RoomStore, db_conn, "presence_stream", "stream_id" ) - self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) - self._state_groups_id_gen = IdGenerator("state_groups", "id", self) - self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) - self._refresh_tokens_id_gen = IdGenerator("refresh_tokens", "id", self) - self._pushers_id_gen = IdGenerator("pushers", "id", self) - self._push_rule_id_gen = IdGenerator("push_rules", "id", self) - self._push_rules_enable_id_gen = IdGenerator("push_rules_enable", "id", self) + self._transaction_id_gen = IdGenerator(db_conn, "sent_transactions", "id") + self._state_groups_id_gen = IdGenerator(db_conn, "state_groups", "id") + self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id") + self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id") + self._pushers_id_gen = IdGenerator(db_conn, "pushers", "id") + self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") + self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") events_max = self._stream_id_gen.get_max_token() event_cache_prefill, min_event_val = self._get_cache_dict( diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 91cbf399b6..21a3240d9d 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -163,12 +163,12 @@ class AccountDataStore(SQLBaseStore): ) self._update_max_stream_id(txn, next_id) - with (yield self._account_data_id_gen.get_next(self)) as next_id: + with self._account_data_id_gen.get_next() as next_id: yield self.runInteraction( "add_room_account_data", add_account_data_txn, next_id ) - result = yield self._account_data_id_gen.get_max_token() + result = self._account_data_id_gen.get_max_token() defer.returnValue(result) @defer.inlineCallbacks @@ -202,12 +202,12 @@ class AccountDataStore(SQLBaseStore): ) self._update_max_stream_id(txn, next_id) - with (yield self._account_data_id_gen.get_next(self)) as next_id: + with self._account_data_id_gen.get_next() as next_id: yield self.runInteraction( "add_user_account_data", add_account_data_txn, next_id ) - result = yield self._account_data_id_gen.get_max_token() + result = self._account_data_id_gen.get_max_token() defer.returnValue(result) def _update_max_stream_id(self, txn, next_id): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 1dd3236829..73a152bc07 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -75,8 +75,8 @@ class EventsStore(SQLBaseStore): yield stream_orderings stream_ordering_manager = stream_ordering_manager() else: - stream_ordering_manager = yield self._stream_id_gen.get_next_mult( - self, len(events_and_contexts) + stream_ordering_manager = self._stream_id_gen.get_next_mult( + len(events_and_contexts) ) with stream_ordering_manager as stream_orderings: @@ -109,7 +109,7 @@ class EventsStore(SQLBaseStore): stream_ordering = self.min_stream_token if stream_ordering is None: - stream_ordering_manager = yield self._stream_id_gen.get_next(self) + stream_ordering_manager = self._stream_id_gen.get_next() else: @contextmanager def stream_ordering_manager(): diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 3ef91d34db..eece7f8961 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -58,8 +58,8 @@ class UserPresenceState(namedtuple("UserPresenceState", class PresenceStore(SQLBaseStore): @defer.inlineCallbacks def update_presence(self, presence_states): - stream_ordering_manager = yield self._presence_id_gen.get_next_mult( - self, len(presence_states) + stream_ordering_manager = self._presence_id_gen.get_next_mult( + len(presence_states) ) with stream_ordering_manager as stream_orderings: diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index bb5c14d912..56e69495b1 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -226,7 +226,7 @@ class PushRuleStore(SQLBaseStore): if txn.rowcount == 0: # We didn't update a row with the given rule_id so insert one - push_rule_id = self._push_rule_id_gen.get_next_txn(txn) + push_rule_id = self._push_rule_id_gen.get_next() self._simple_insert_txn( txn, @@ -279,7 +279,7 @@ class PushRuleStore(SQLBaseStore): defer.returnValue(ret) def _set_push_rule_enabled_txn(self, txn, user_id, rule_id, enabled): - new_id = self._push_rules_enable_id_gen.get_next_txn(txn) + new_id = self._push_rules_enable_id_gen.get_next() self._simple_upsert_txn( txn, "push_rules_enable", diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index c23648cdbc..7693ab9082 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -84,7 +84,7 @@ class PusherStore(SQLBaseStore): app_display_name, device_display_name, pushkey, pushkey_ts, lang, data, profile_tag=""): try: - next_id = yield self._pushers_id_gen.get_next() + next_id = self._pushers_id_gen.get_next() yield self._simple_upsert( "pushers", dict( diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index a7343c97f7..cd6dca4901 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -330,7 +330,7 @@ class ReceiptsStore(SQLBaseStore): "insert_receipt_conv", graph_to_linear ) - stream_id_manager = yield self._receipts_id_gen.get_next(self) + stream_id_manager = self._receipts_id_gen.get_next() with stream_id_manager as stream_id: have_persisted = yield self.runInteraction( "insert_linearized_receipt", @@ -347,7 +347,7 @@ class ReceiptsStore(SQLBaseStore): room_id, receipt_type, user_id, event_ids, data ) - max_persisted_id = yield self._stream_id_gen.get_max_token() + max_persisted_id = self._stream_id_gen.get_max_token() defer.returnValue((stream_id, max_persisted_id)) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 03a9b66e4a..ad1157f979 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -40,7 +40,7 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - next_id = yield self._access_tokens_id_gen.get_next() + next_id = self._access_tokens_id_gen.get_next() yield self._simple_insert( "access_tokens", @@ -62,7 +62,7 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - next_id = yield self._refresh_tokens_id_gen.get_next() + next_id = self._refresh_tokens_id_gen.get_next() yield self._simple_insert( "refresh_tokens", @@ -99,7 +99,7 @@ class RegistrationStore(SQLBaseStore): def _register(self, txn, user_id, token, password_hash, was_guest, make_guest): now = int(self.clock.time()) - next_id = self._access_tokens_id_gen.get_next_txn(txn) + next_id = self._access_tokens_id_gen.get_next() try: if was_guest: diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 372b540002..8ed8a21b0a 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -83,7 +83,7 @@ class StateStore(SQLBaseStore): if event.is_state(): state_events[(event.type, event.state_key)] = event - state_group = self._state_groups_id_gen.get_next_txn(txn) + state_group = self._state_groups_id_gen.get_next() self._simple_insert_txn( txn, table="state_groups", diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py index 9551aa9739..1127b0bd7e 100644 --- a/synapse/storage/tags.py +++ b/synapse/storage/tags.py @@ -142,12 +142,12 @@ class TagsStore(SQLBaseStore): ) self._update_revision_txn(txn, user_id, room_id, next_id) - with (yield self._account_data_id_gen.get_next(self)) as next_id: + with self._account_data_id_gen.get_next() as next_id: yield self.runInteraction("add_tag", add_tag_txn, next_id) self.get_tags_for_user.invalidate((user_id,)) - result = yield self._account_data_id_gen.get_max_token() + result = self._account_data_id_gen.get_max_token() defer.returnValue(result) @defer.inlineCallbacks @@ -164,12 +164,12 @@ class TagsStore(SQLBaseStore): txn.execute(sql, (user_id, room_id, tag)) self._update_revision_txn(txn, user_id, room_id, next_id) - with (yield self._account_data_id_gen.get_next(self)) as next_id: + with self._account_data_id_gen.get_next() as next_id: yield self.runInteraction("remove_tag", remove_tag_txn, next_id) self.get_tags_for_user.invalidate((user_id,)) - result = yield self._account_data_id_gen.get_max_token() + result = self._account_data_id_gen.get_max_token() defer.returnValue(result) def _update_revision_txn(self, txn, user_id, room_id, next_id): diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 4475c451c1..d338dfcf0a 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -117,7 +117,7 @@ class TransactionStore(SQLBaseStore): def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts): - next_id = self._transaction_id_gen.get_next_txn(txn) + next_id = self._transaction_id_gen.get_next() # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index ef5e4a4668..efe3f68e6e 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -13,51 +13,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from collections import deque import contextlib import threading class IdGenerator(object): - def __init__(self, table, column, store): + def __init__(self, db_conn, table, column): self.table = table self.column = column - self.store = store self._lock = threading.Lock() - self._next_id = None + cur = db_conn.cursor() + self._next_id = self._load_next_id(cur) + cur.close() - @defer.inlineCallbacks - def get_next(self): - if self._next_id is None: - yield self.store.runInteraction( - "IdGenerator_%s" % (self.table,), - self.get_next_txn, - ) + def _load_next_id(self, txn): + txn.execute("SELECT MAX(%s) FROM %s" % (self.column, self.table,)) + val, = txn.fetchone() + return val + 1 if val else 1 + def get_next(self): with self._lock: i = self._next_id self._next_id += 1 - defer.returnValue(i) - - def get_next_txn(self, txn): - with self._lock: - if self._next_id: - i = self._next_id - self._next_id += 1 - return i - else: - txn.execute( - "SELECT MAX(%s) FROM %s" % (self.column, self.table,) - ) - - val, = txn.fetchone() - cur = val or 0 - cur += 1 - self._next_id = cur + 1 - - return cur + return i class StreamIdGenerator(object): @@ -69,7 +48,7 @@ class StreamIdGenerator(object): persistence of events can complete out of order. Usage: - with stream_id_gen.get_next_txn(txn) as stream_id: + with stream_id_gen.get_next() as stream_id: # ... persist event ... """ def __init__(self, db_conn, table, column): @@ -79,15 +58,21 @@ class StreamIdGenerator(object): self._lock = threading.Lock() cur = db_conn.cursor() - self._current_max = self._get_or_compute_current_max(cur) + self._current_max = self._load_current_max(cur) cur.close() self._unfinished_ids = deque() - def get_next(self, store): + def _load_current_max(self, txn): + txn.execute("SELECT MAX(%s) FROM %s" % (self.column, self.table)) + rows = txn.fetchall() + val, = rows[0] + return int(val) if val else 1 + + def get_next(self): """ Usage: - with yield stream_id_gen.get_next as stream_id: + with stream_id_gen.get_next() as stream_id: # ... persist event ... """ with self._lock: @@ -106,10 +91,10 @@ class StreamIdGenerator(object): return manager() - def get_next_mult(self, store, n): + def get_next_mult(self, n): """ Usage: - with yield stream_id_gen.get_next(store, n) as stream_ids: + with stream_id_gen.get_next(n) as stream_ids: # ... persist events ... """ with self._lock: @@ -139,13 +124,3 @@ class StreamIdGenerator(object): return self._unfinished_ids[0] - 1 return self._current_max - - def _get_or_compute_current_max(self, txn): - with self._lock: - txn.execute("SELECT MAX(%s) FROM %s" % (self.column, self.table)) - rows = txn.fetchall() - val, = rows[0] - - self._current_max = int(val) if val else 1 - - return self._current_max -- cgit 1.5.1 From 465605d616c991760ce021932f0453fc6bc477ef Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Mar 2016 15:58:22 +0000 Subject: Store appservice ID on register --- synapse/handlers/register.py | 5 ++++- synapse/storage/registration.py | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 7 deletions(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index e2ace6a4e5..6ffb8c0da6 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -182,6 +182,8 @@ class RegistrationHandler(BaseHandler): errcode=Codes.EXCLUSIVE ) + service_id = service.id if service.is_exclusive_user(user_id) else None + yield self.check_user_id_not_appservice_exclusive( user_id, allowed_appservice=service ) @@ -190,7 +192,8 @@ class RegistrationHandler(BaseHandler): yield self.store.register( user_id=user_id, token=token, - password_hash="" + password_hash="", + appservice_id=service_id, ) yield registered_user(self.distributor, user) defer.returnValue((user_id, token)) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index ad1157f979..aa49f53458 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -76,7 +76,7 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def register(self, user_id, token, password_hash, - was_guest=False, make_guest=False): + was_guest=False, make_guest=False, appservice_id=None): """Attempts to register an account. Args: @@ -87,16 +87,32 @@ class RegistrationStore(SQLBaseStore): upgraded to a non-guest account. make_guest (boolean): True if the the new user should be guest, false to add a regular user account. + appservice_id (str): The ID of the appservice registering the user. Raises: StoreError if the user_id could not be registered. """ yield self.runInteraction( "register", - self._register, user_id, token, password_hash, was_guest, make_guest + self._register, + user_id, + token, + password_hash, + was_guest, + make_guest, + appservice_id ) self.is_guest.invalidate((user_id,)) - def _register(self, txn, user_id, token, password_hash, was_guest, make_guest): + def _register( + self, + txn, + user_id, + token, + password_hash, + was_guest, + make_guest, + appservice_id + ): now = int(self.clock.time()) next_id = self._access_tokens_id_gen.get_next() @@ -111,9 +127,21 @@ class RegistrationStore(SQLBaseStore): [password_hash, now, 1 if make_guest else 0, user_id]) else: txn.execute("INSERT INTO users " - "(name, password_hash, creation_ts, is_guest) " - "VALUES (?,?,?,?)", - [user_id, password_hash, now, 1 if make_guest else 0]) + "(" + " name," + " password_hash," + " creation_ts," + " is_guest," + " appservice_id" + ") " + "VALUES (?,?,?,?,?)", + [ + user_id, + password_hash, + now, + 1 if make_guest else 0, + appservice_id, + ]) except self.database_engine.module.IntegrityError: raise StoreError( 400, "User ID already taken.", errcode=Codes.USER_IN_USE -- cgit 1.5.1 From aa11db5f119b9fa88242b0df95cfddd00e196ca1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 11 Mar 2016 13:14:18 +0000 Subject: Fix cache invalidation so deleting access tokens (which we did when changing password) actually takes effect without HS restart. Reinstate the code to avoid logging out the session that changed the password, removed in 415c2f05491ce65a4fc34326519754cd1edd9c54 --- synapse/handlers/auth.py | 13 +++++++++---- synapse/push/pusherpool.py | 8 ++++---- synapse/rest/client/v2_alpha/account.py | 2 +- synapse/storage/registration.py | 28 ++++++++++++++++++++-------- 4 files changed, 34 insertions(+), 17 deletions(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 7a4afe446d..a740cc3da3 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -432,13 +432,18 @@ class AuthHandler(BaseHandler): ) @defer.inlineCallbacks - def set_password(self, user_id, newpassword): + def set_password(self, user_id, newpassword, requester=None): password_hash = self.hash(newpassword) + except_access_token_ids = [requester.access_token_id] if requester else [] + yield self.store.user_set_password_hash(user_id, password_hash) - yield self.store.user_delete_access_tokens(user_id) - yield self.hs.get_pusherpool().remove_pushers_by_user(user_id) - yield self.store.flush_user(user_id) + yield self.store.user_delete_access_tokens_except( + user_id, except_access_token_ids + ) + yield self.hs.get_pusherpool().remove_pushers_by_user_except_access_tokens( + user_id, except_access_token_ids + ) @defer.inlineCallbacks def add_threepid(self, user_id, medium, address, validated_at): diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 772a095f8b..28ec94d866 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -92,14 +92,14 @@ class PusherPool: yield self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) @defer.inlineCallbacks - def remove_pushers_by_user(self, user_id): + def remove_pushers_by_user_except_access_tokens(self, user_id, except_token_ids): all = yield self.store.get_all_pushers() logger.info( - "Removing all pushers for user %s", - user_id, + "Removing all pushers for user %s except access tokens ids %r", + user_id, except_token_ids ) for p in all: - if p['user_name'] == user_id: + if p['user_name'] == user_id and p['access_token'] not in except_token_ids: logger.info( "Removing pusher for app id %s, pushkey %s, user %s", p['app_id'], p['pushkey'], p['user_name'] diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 688b051580..dd4ea45588 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -79,7 +79,7 @@ class PasswordRestServlet(RestServlet): new_password = params['new_password'] yield self.auth_handler.set_password( - user_id, new_password + user_id, new_password, requester ) defer.returnValue((200, {})) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index aa49f53458..5eef7ebcc7 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -208,14 +208,26 @@ class RegistrationStore(SQLBaseStore): ) @defer.inlineCallbacks - def flush_user(self, user_id): - rows = yield self._execute( - 'flush_user', None, - "SELECT token FROM access_tokens WHERE user_id = ?", - user_id - ) - for r in rows: - self.get_user_by_access_token.invalidate((r,)) + def user_delete_access_tokens_except(self, user_id, except_token_ids): + def f(txn): + txn.execute( + "SELECT id, token FROM access_tokens WHERE user_id = ? LIMIT 50", + (user_id,) + ) + rows = txn.fetchall() + for r in rows: + if r[0] in except_token_ids: + continue + + txn.call_after(self.get_user_by_access_token.invalidate, (r[1],)) + txn.execute( + "DELETE FROM access_tokens WHERE id in (%s)" % ",".join( + ["?" for _ in rows] + ), [r[0] for r in rows] + ) + return len(rows) == 50 + while (yield self.runInteraction("user_delete_access_tokens_except", f)): + pass @cached() def get_user_by_access_token(self, token): -- cgit 1.5.1 From 57c444b3ad9e69ae99fe694c0ef9a1961ec9366a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 11 Mar 2016 14:25:05 +0000 Subject: Dear PyCharm, please indent sensibly for me. Thx. --- synapse/handlers/auth.py | 4 ++-- synapse/storage/registration.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index a740cc3da3..0f02493fa2 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -439,10 +439,10 @@ class AuthHandler(BaseHandler): yield self.store.user_set_password_hash(user_id, password_hash) yield self.store.user_delete_access_tokens_except( - user_id, except_access_token_ids + user_id, except_access_token_ids ) yield self.hs.get_pusherpool().remove_pushers_by_user_except_access_tokens( - user_id, except_access_token_ids + user_id, except_access_token_ids ) @defer.inlineCallbacks diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 5eef7ebcc7..d3d84c9b94 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -212,7 +212,7 @@ class RegistrationStore(SQLBaseStore): def f(txn): txn.execute( "SELECT id, token FROM access_tokens WHERE user_id = ? LIMIT 50", - (user_id,) + (user_id,) ) rows = txn.fetchall() for r in rows: -- cgit 1.5.1 From f523177850df7fbe480086b281f09815f3d5c656 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 11 Mar 2016 14:29:01 +0000 Subject: Delete old, unused methods and rename new one to just be `user_delete_access_tokens` with an `except_token_ids` argument doing what it says on the tin. --- synapse/handlers/auth.py | 2 +- synapse/storage/registration.py | 17 ++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 0f02493fa2..43f2cdc2c4 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -438,7 +438,7 @@ class AuthHandler(BaseHandler): except_access_token_ids = [requester.access_token_id] if requester else [] yield self.store.user_set_password_hash(user_id, password_hash) - yield self.store.user_delete_access_tokens_except( + yield self.store.user_delete_access_tokens( user_id, except_access_token_ids ) yield self.hs.get_pusherpool().remove_pushers_by_user_except_access_tokens( diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index d3d84c9b94..266e29f5bc 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -195,20 +195,7 @@ class RegistrationStore(SQLBaseStore): }) @defer.inlineCallbacks - def user_delete_access_tokens(self, user_id): - yield self.runInteraction( - "user_delete_access_tokens", - self._user_delete_access_tokens, user_id - ) - - def _user_delete_access_tokens(self, txn, user_id): - txn.execute( - "DELETE FROM access_tokens WHERE user_id = ?", - (user_id, ) - ) - - @defer.inlineCallbacks - def user_delete_access_tokens_except(self, user_id, except_token_ids): + def user_delete_access_tokens(self, user_id, except_token_ids): def f(txn): txn.execute( "SELECT id, token FROM access_tokens WHERE user_id = ? LIMIT 50", @@ -226,7 +213,7 @@ class RegistrationStore(SQLBaseStore): ), [r[0] for r in rows] ) return len(rows) == 50 - while (yield self.runInteraction("user_delete_access_tokens_except", f)): + while (yield self.runInteraction("user_delete_access_tokens", f)): pass @cached() -- cgit 1.5.1 From af59826a2fda31a6382f60659796a1c8db6f21ce Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 11 Mar 2016 14:34:09 +0000 Subject: Make select more sensible when dseleting access tokens, rename pusher deletion to match access token deletion and make exception arg optional. --- synapse/handlers/auth.py | 2 +- synapse/push/pusherpool.py | 2 +- synapse/storage/registration.py | 8 +++----- 3 files changed, 5 insertions(+), 7 deletions(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 43f2cdc2c4..5c0ea636bc 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -441,7 +441,7 @@ class AuthHandler(BaseHandler): yield self.store.user_delete_access_tokens( user_id, except_access_token_ids ) - yield self.hs.get_pusherpool().remove_pushers_by_user_except_access_tokens( + yield self.hs.get_pusherpool().remove_pushers_by_user( user_id, except_access_token_ids ) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 28ec94d866..0b463c6fdb 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -92,7 +92,7 @@ class PusherPool: yield self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) @defer.inlineCallbacks - def remove_pushers_by_user_except_access_tokens(self, user_id, except_token_ids): + def remove_pushers_by_user(self, user_id, except_token_ids=[]): all = yield self.store.get_all_pushers() logger.info( "Removing all pushers for user %s except access tokens ids %r", diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 266e29f5bc..3a050621d9 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -198,14 +198,12 @@ class RegistrationStore(SQLBaseStore): def user_delete_access_tokens(self, user_id, except_token_ids): def f(txn): txn.execute( - "SELECT id, token FROM access_tokens WHERE user_id = ? LIMIT 50", - (user_id,) + "SELECT id, token FROM access_tokens " + "WHERE user_id = ? AND id not in LIMIT 50", + (user_id,except_token_ids) ) rows = txn.fetchall() for r in rows: - if r[0] in except_token_ids: - continue - txn.call_after(self.get_user_by_access_token.invalidate, (r[1],)) txn.execute( "DELETE FROM access_tokens WHERE id in (%s)" % ",".join( -- cgit 1.5.1 From 2dee03aee513eddaf50a4249747beac67445b3cd Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 11 Mar 2016 14:38:23 +0000 Subject: more pep8 --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 3a050621d9..5d45f0c651 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -200,7 +200,7 @@ class RegistrationStore(SQLBaseStore): txn.execute( "SELECT id, token FROM access_tokens " "WHERE user_id = ? AND id not in LIMIT 50", - (user_id,except_token_ids) + (user_id, except_token_ids) ) rows = txn.fetchall() for r in rows: -- cgit 1.5.1 From c08122843982307dad8d099784bc1a4bc0202eae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Mar 2016 15:09:17 +0000 Subject: Fix SQL statement --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 5d45f0c651..5e7a4e371d 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -199,7 +199,7 @@ class RegistrationStore(SQLBaseStore): def f(txn): txn.execute( "SELECT id, token FROM access_tokens " - "WHERE user_id = ? AND id not in LIMIT 50", + "WHERE user_id = ? AND id NOT IN ? LIMIT 50", (user_id, except_token_ids) ) rows = txn.fetchall() -- cgit 1.5.1 From b13035cc91410634421820e5175d0596f5a67549 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Mar 2016 16:27:50 +0000 Subject: Implement logout --- synapse/rest/__init__.py | 2 ++ synapse/rest/client/v1/logout.py | 72 ++++++++++++++++++++++++++++++++++++++++ synapse/storage/registration.py | 49 +++++++++++++++++++-------- 3 files changed, 109 insertions(+), 14 deletions(-) create mode 100644 synapse/rest/client/v1/logout.py (limited to 'synapse/storage/registration.py') diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 433237c204..6688fa8fa0 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -30,6 +30,7 @@ from synapse.rest.client.v1 import ( push_rule, register as v1_register, login as v1_login, + logout, ) from synapse.rest.client.v2_alpha import ( @@ -72,6 +73,7 @@ class ClientRestResource(JsonResource): admin.register_servlets(hs, client_resource) pusher.register_servlets(hs, client_resource) push_rule.register_servlets(hs, client_resource) + logout.register_servlets(hs, client_resource) # "v2" sync.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v1/logout.py b/synapse/rest/client/v1/logout.py new file mode 100644 index 0000000000..9bff02ee4e --- /dev/null +++ b/synapse/rest/client/v1/logout.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import AuthError, Codes + +from .base import ClientV1RestServlet, client_path_patterns + +import logging + + +logger = logging.getLogger(__name__) + + +class LogoutRestServlet(ClientV1RestServlet): + PATTERNS = client_path_patterns("/logout$") + + def __init__(self, hs): + super(LogoutRestServlet, self).__init__(hs) + self.store = hs.get_datastore() + + def on_OPTIONS(self, request): + return (200, {}) + + @defer.inlineCallbacks + def on_POST(self, request): + try: + access_token = request.args["access_token"][0] + except KeyError: + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", + errcode=Codes.MISSING_TOKEN + ) + yield self.store.delete_access_token(access_token) + defer.returnValue((200, {})) + + +class LogoutAllRestServlet(ClientV1RestServlet): + PATTERNS = client_path_patterns("/logout/all$") + + def __init__(self, hs): + super(LogoutAllRestServlet, self).__init__(hs) + self.store = hs.get_datastore() + self.auth = hs.get_auth() + + def on_OPTIONS(self, request): + return (200, {}) + + @defer.inlineCallbacks + def on_POST(self, request): + requester = yield self.auth.get_user_by_req(request) + user_id = requester.user.to_string() + yield self.store.user_delete_access_tokens(user_id) + defer.returnValue((200, {})) + + +def register_servlets(hs, http_server): + LogoutRestServlet(hs).register(http_server) + LogoutAllRestServlet(hs).register(http_server) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 5e7a4e371d..18898c44eb 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -195,24 +195,45 @@ class RegistrationStore(SQLBaseStore): }) @defer.inlineCallbacks - def user_delete_access_tokens(self, user_id, except_token_ids): + def user_delete_access_tokens(self, user_id, except_token_ids=[]): def f(txn): txn.execute( - "SELECT id, token FROM access_tokens " - "WHERE user_id = ? AND id NOT IN ? LIMIT 50", - (user_id, except_token_ids) + "SELECT token FROM access_tokens" + " WHERE user_id = ? AND id NOT IN (%s)" % ( + ",".join(["?" for _ in except_token_ids]), + ), + [user_id] + except_token_ids ) - rows = txn.fetchall() - for r in rows: - txn.call_after(self.get_user_by_access_token.invalidate, (r[1],)) - txn.execute( - "DELETE FROM access_tokens WHERE id in (%s)" % ",".join( - ["?" for _ in rows] - ), [r[0] for r in rows] + + while True: + rows = txn.fetchmany(100) + if not rows: + break + + for row in rows: + txn.call_after(self.get_user_by_access_token.invalidate, (row[0],)) + + txn.execute( + "DELETE FROM access_tokens WHERE token in (%s)" % ( + ",".join(["?" for _ in rows]), + ), [r[0] for r in rows] + ) + + yield self.runInteraction("user_delete_access_tokens", f) + + def delete_access_token(self, access_token): + def f(txn): + self._simple_delete_one_txn( + txn, + table="access_tokens", + keyvalues={ + "token": access_token + }, ) - return len(rows) == 50 - while (yield self.runInteraction("user_delete_access_tokens", f)): - pass + + txn.call_after(self.get_user_by_access_token.invalidate, (access_token,)) + + return self.runInteraction("delete_access_token", f) @cached() def get_user_by_access_token(self, token): -- cgit 1.5.1 From 15122da0e275ba18ec4633129715067a637f38af Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Mar 2016 16:45:27 +0000 Subject: Thats not how transactions work. --- synapse/storage/registration.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'synapse/storage/registration.py') diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 18898c44eb..bd4eb88a92 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -197,26 +197,29 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def user_delete_access_tokens(self, user_id, except_token_ids=[]): def f(txn): - txn.execute( - "SELECT token FROM access_tokens" - " WHERE user_id = ? AND id NOT IN (%s)" % ( + sql = "SELECT token FROM access_tokens WHERE user_id = ?" + clauses = [user_id] + + if except_token_ids: + sql += " AND id NOT IN (%s)" % ( ",".join(["?" for _ in except_token_ids]), - ), - [user_id] + except_token_ids - ) + ) + clauses += except_token_ids + + txn.execute(sql, clauses) - while True: - rows = txn.fetchmany(100) - if not rows: - break + rows = txn.fetchall() - for row in rows: + n = 100 + chunks = [rows[i:i + n] for i in xrange(0, len(rows), n)] + for chunk in chunks: + for row in chunk: txn.call_after(self.get_user_by_access_token.invalidate, (row[0],)) txn.execute( "DELETE FROM access_tokens WHERE token in (%s)" % ( - ",".join(["?" for _ in rows]), - ), [r[0] for r in rows] + ",".join(["?" for _ in chunk]), + ), [r[0] for r in chunk] ) yield self.runInteraction("user_delete_access_tokens", f) -- cgit 1.5.1