From 3737329d9b40b54bc4205c5f5e7e0946c5e51614 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 10:53:18 +0000 Subject: Handle the fact the list.remove raises if element doesn't exist --- synapse/handlers/federation.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index aba266c2bc..9d0ce9aa50 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -332,8 +332,11 @@ class FederationHandler(BaseHandler): # Try the host we successfully got a response to /make_join/ # request first. - target_hosts.remove(origin) - target_hosts.insert(0, origin) + try: + target_hosts.remove(origin) + target_hosts.insert(0, origin) + except ValueError: + pass ret = yield self.replication_layer.send_join( target_hosts, @@ -521,7 +524,7 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) - destinations.remove(origin) + destinations.discard(origin) logger.debug( "on_send_join_request: Sending event: %s, signatures: %s", @@ -1013,7 +1016,10 @@ class FederationHandler(BaseHandler): for e in missing_remotes: for e_id, _ in e.auth_events: if e_id in missing_remote_ids: - base_remote_rejected.remove(e) + try: + base_remote_rejected.remove(e) + except ValueError: + pass reason_map = {} -- cgit 1.4.1 From c78b5fb1f16957f5398517fa5a99dcb370992b26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 13:52:16 +0000 Subject: Make seen_ids a set --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9d0ce9aa50..1b709e5e24 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -130,7 +130,9 @@ class FederationHandler(BaseHandler): if auth_chain: event_ids |= {e.event_id for e in auth_chain} - seen_ids = (yield self.store.have_events(event_ids)).keys() + seen_ids = set( + (yield self.store.have_events(event_ids)).keys() + ) if state and auth_chain is not None: # If we have any state or auth_chain given to us by the replication -- cgit 1.4.1 From e890ce223c0c7798efc9961ca1e4c4870cfc7e2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 14:16:50 +0000 Subject: Don't query auth if the only difference is events that were rejected due to auth. --- synapse/handlers/federation.py | 128 +++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 56 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1b709e5e24..b13b7c7701 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -791,12 +791,12 @@ class FederationHandler(BaseHandler): @log_function def do_auth(self, origin, event, context, auth_events): # Check if we have all the auth events. - res = yield self.store.have_events( + have_events = yield self.store.have_events( [e_id for e_id, _ in event.auth_events] ) event_auth_events = set(e_id for e_id, _ in event.auth_events) - seen_events = set(res.keys()) + seen_events = set(have_events.keys()) missing_auth = event_auth_events - seen_events @@ -839,6 +839,11 @@ class FederationHandler(BaseHandler): auth_events[(e.type, e.state_key)] = e except AuthError: pass + + have_events = yield self.store.have_events( + [e_id for e_id, _ in event.auth_events] + ) + seen_events = set(have_events.keys()) except: # FIXME: logger.exception("Failed to get auth chain") @@ -852,64 +857,75 @@ class FederationHandler(BaseHandler): # Do auth conflict res. logger.debug("Different auth: %s", different_auth) - # 1. Get what we think is the auth chain. - auth_ids = self.auth.compute_auth_events( - event, context.current_state - ) - local_auth_chain = yield self.store.get_auth_chain(auth_ids) - - try: - # 2. Get remote difference. - result = yield self.replication_layer.query_auth( - origin, - event.room_id, - event.event_id, - local_auth_chain, - ) - - seen_remotes = yield self.store.have_events( - [e.event_id for e in result["auth_chain"]] + # Only do auth resolution if we have something new to say. + # We can't rove an auth failure. + do_resolution = False + for e_id in different_auth: + if e_id in have_events: + if have_events[e_id] != RejectedReason.AUTH_ERROR: + do_resolution = True + break + + if do_resolution: + # 1. Get what we think is the auth chain. + auth_ids = self.auth.compute_auth_events( + event, context.current_state ) + local_auth_chain = yield self.store.get_auth_chain(auth_ids) - # 3. Process any remote auth chain events we haven't seen. - for ev in result["auth_chain"]: - if ev.event_id in seen_remotes.keys(): - continue - - if ev.event_id == event.event_id: - continue - - try: - auth_ids = [e_id for e_id, _ in ev.auth_events] - auth = { - (e.type, e.state_key): e for e in result["auth_chain"] - if e.event_id in auth_ids - } - ev.internal_metadata.outlier = True - - logger.debug( - "do_auth %s different_auth: %s", - event.event_id, e.event_id - ) + try: + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) - yield self._handle_new_event( - origin, ev, auth_events=auth - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) - if ev.event_id in event_auth_events: - auth_events[(ev.type, ev.state_key)] = ev - except AuthError: - pass + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue + + if ev.event_id == event.event_id: + continue + + try: + auth_ids = [e_id for e_id, _ in ev.auth_events] + auth = { + (e.type, e.state_key): e + for e in result["auth_chain"] + if e.event_id in auth_ids + } + ev.internal_metadata.outlier = True + + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) + + yield self._handle_new_event( + origin, ev, auth_events=auth + ) + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev + except AuthError: + pass - except: - # FIXME: - logger.exception("Failed to query auth chain") + except: + # FIXME: + logger.exception("Failed to query auth chain") - # 4. Look at rejects and their proofs. - # TODO. + # 4. Look at rejects and their proofs. + # TODO. - context.current_state.update(auth_events) - context.state_group = None + context.current_state.update(auth_events) + context.state_group = None try: self.auth.check(event, auth_events=auth_events) @@ -1028,9 +1044,9 @@ class FederationHandler(BaseHandler): for e in base_remote_rejected: reason = yield self.store.get_rejection_reason(e.event_id) if reason is None: - # FIXME: ERRR?! - logger.warn("Could not find reason for %s", e.event_id) - raise RuntimeError("Could not find reason for %s" % e.event_id) + # TODO: e is not in the current state, so we should + # construct some proof of that. + continue reason_map[e.event_id] = reason -- cgit 1.4.1 From 55a186485c8def40e578201bb33457c9b7df9dde Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 6 Feb 2015 15:58:40 +0000 Subject: SYN-258: get_recent_events_for_room only accepts stream tokens, convert the topological token to a stream token before passing it to get_recent_events_for_room --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 962686f4bb..439164ae39 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,15 +298,17 @@ class SyncHandler(BaseHandler): load_limit = max(sync_config.limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key + end_key = room_key while limited and len(recents) < sync_config.limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, - end_token=room_key, + end_token=end_key, ) (room_key, _) = keys + end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents -- cgit 1.4.1 From 582019f870adbc4a8a8a9ef97b527e0fead77761 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 13:32:14 +0000 Subject: ...and here's the actual impl. git fail. --- synapse/handlers/register.py | 14 ++++++++ synapse/rest/media/v1/upload_resource.py | 57 ++++++++++++++++++-------------- 2 files changed, 46 insertions(+), 25 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 66a89c10b2..2b9d860084 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -99,6 +99,20 @@ class RegistrationHandler(BaseHandler): raise RegistrationError( 500, "Cannot generate user ID.") + # create a default avatar for the user + # XXX: ideally clients would explicitly specify one, but given they don't + # and we want consistent and pretty identicons for random users, we'll + # do it here. + auth_user = UserID.from_string(user_id) + identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) + upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) + content_uri = yield upload_resource.create_content( + "image/png", None, identicon_bytes, len(identicon_bytes), auth_user + ) + profile_handler = self.hs.get_handlers().profile_handler + profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index b939a30e19..5b42782331 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -38,6 +38,35 @@ class UploadResource(BaseMediaResource): def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET + + @defer.inlineCallbacks + def create_content(self, media_type, upload_name, content, content_length, auth_user): + media_id = random_string(24) + + fname = self.filepaths.local_media_filepath(media_id) + self._makedirs(fname) + + # This shouldn't block for very long because the content will have + # already been uploaded at this point. + with open(fname, "wb") as f: + f.write(content) + + yield self.store.store_local_media( + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + ) + media_info = { + "media_type": media_type, + "media_length": content_length, + } + + yield self._generate_local_thumbnails(media_id, media_info) + + defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @defer.inlineCallbacks def _async_render_POST(self, request): @@ -70,32 +99,10 @@ class UploadResource(BaseMediaResource): # disposition = headers.getRawHeaders("Content-Disposition")[0] # TODO(markjh): parse content-dispostion - media_id = random_string(24) - - fname = self.filepaths.local_media_filepath(media_id) - self._makedirs(fname) - - # This shouldn't block for very long because the content will have - # already been uploaded at this point. - with open(fname, "wb") as f: - f.write(request.content.read()) - - yield self.store.store_local_media( - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=None, - media_length=content_length, - user_id=auth_user, + content_uri = yield self.create_content( + media_type, None, request.content.read(), + content_length, auth_user ) - media_info = { - "media_type": media_type, - "media_length": content_length, - } - - yield self._generate_local_thumbnails(media_id, media_info) - - content_uri = "mxc://%s/%s" % (self.server_name, media_id) respond_with_json( request, 200, {"content_uri": content_uri}, send_cors=True -- cgit 1.4.1 From 37b6b880ef084f2cdd3cce1a275aa46764131c91 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 21:24:08 +0000 Subject: don't give up if we can't create default avatars during tests --- synapse/handlers/register.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 2b9d860084..4f06c487b1 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -103,15 +103,18 @@ class RegistrationHandler(BaseHandler): # XXX: ideally clients would explicitly specify one, but given they don't # and we want consistent and pretty identicons for random users, we'll # do it here. - auth_user = UserID.from_string(user_id) - identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) - upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) - identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) - content_uri = yield upload_resource.create_content( - "image/png", None, identicon_bytes, len(identicon_bytes), auth_user - ) - profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + try: + auth_user = UserID.from_string(user_id) + identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) + upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) + content_uri = yield upload_resource.create_content( + "image/png", None, identicon_bytes, len(identicon_bytes), auth_user + ) + profile_handler = self.hs.get_handlers().profile_handler + profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + except NotImplementedError: + pass # make tests pass without messing around creating default avatars defer.returnValue((user_id, token)) -- cgit 1.4.1