diff options
| author | Erik Johnston <erik@matrix.org> | 2015-01-07 14:34:00 +0000 |
|---|---|---|
| committer | Erik Johnston <erik@matrix.org> | 2015-01-07 14:34:00 +0000 |
| commit | bacaa215eb2e1c11e50bbc3fb0e43e28a463458b (patch) | |
| tree | 27510a6358b8bc219ccd456622b963a3b394bee3 /synapse | |
| parent | Merge branch 'hotfixes-v0.6.0a' (diff) | |
| parent | Improve change log (diff) | |
| download | synapse-bacaa215eb2e1c11e50bbc3fb0e43e28a463458b.tar.xz | |
Merge branch 'release-v0.6.1' of github.com:matrix-org/synapse v0.6.1
Diffstat (limited to 'synapse')
132 files changed, 580 insertions, 307 deletions
diff --git a/synapse/__init__.py b/synapse/__init__.py index 1cef40925f..c3f1ac63be 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.0" +__version__ = "0.6.1" diff --git a/synapse/api/__init__.py b/synapse/api/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/api/__init__.py +++ b/synapse/api/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 70245aba04..e31482cfaa 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 4fc8b79a40..7ee6dcc46e 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e250b9b211..2b049debf3 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index b25358090f..3f9ad4ce89 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/urls.py b/synapse/api/urls.py index d7625127f8..a299392049 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index c78b6f574d..5fec8da7ca 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 52a0b729f4..363c20f994 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/__init__.py b/synapse/config/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/config/__init__.py +++ b/synapse/config/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 1cdd03e414..dfc115d8e8 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 4ed9070b9e..7e21c7414d 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/database.py b/synapse/config/database.py index 0aac8c8382..0d33583a7d 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/email.py b/synapse/config/email.py index 6bab133224..f0854f8c37 100644 --- a/synapse/config/email.py +++ b/synapse/config/email.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 5a11fd6c76..b0fe217459 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/logger.py b/synapse/config/logger.py index d352ea9be9..15383b3184 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index fb63ed7d9b..17c7e64ce7 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/repository.py b/synapse/config/repository.py index f1b7b1b74e..e1827f05e4 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014, 2015 matrix.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ class ContentRepositoryConfig(Config): super(ContentRepositoryConfig, cls).add_arguments(parser) db_group = parser.add_argument_group("content_repository") db_group.add_argument( - "--max-upload-size", default="1M" + "--max-upload-size", default="10M" ) db_group.add_argument( "--media-store-path", default=cls.default_path("media_store") diff --git a/synapse/config/server.py b/synapse/config/server.py index f8a0844b8c..4f73c85466 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 3600c3ea9e..384b29e7ba 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/voip.py b/synapse/config/voip.py index 06675966ce..a2b822719f 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/__init__.py b/synapse/crypto/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/crypto/__init__.py +++ b/synapse/crypto/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 3143322d9c..24d4abf3e9 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 21c19c971d..6633b19565 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 3f37c99261..9c910fa3fc 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index ceb03ce6c2..3fb99f7125 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 34b1b944ab..4252e5ab5c 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,12 +15,10 @@ from synapse.util.frozenutils import freeze, unfreeze -import copy - class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): - self.__dict__ = copy.deepcopy(internal_metadata_dict) + self.__dict__ = internal_metadata_dict def get_dict(self): return dict(self.__dict__) @@ -49,10 +47,10 @@ def _event_dict_property(key): class EventBase(object): def __init__(self, event_dict, signatures={}, unsigned={}, internal_metadata_dict={}): - self.signatures = copy.deepcopy(signatures) - self.unsigned = copy.deepcopy(unsigned) + self.signatures = signatures + self.unsigned = unsigned - self._event_dict = copy.deepcopy(event_dict) + self._event_dict = event_dict self.internal_metadata = _EventInternalMetadata( internal_metadata_dict @@ -112,10 +110,16 @@ class EventBase(object): class FrozenEvent(EventBase): def __init__(self, event_dict, internal_metadata_dict={}): - event_dict = copy.deepcopy(event_dict) + event_dict = dict(event_dict) + + # Signatures is a dict of dicts, and this is faster than doing a + # copy.deepcopy + signatures = { + name: {sig_id: sig for sig_id, sig in sigs.items()} + for name, sigs in event_dict.pop("signatures", {}).items() + } - signatures = copy.deepcopy(event_dict.pop("signatures", {})) - unsigned = copy.deepcopy(event_dict.pop("unsigned", {})) + unsigned = dict(event_dict.pop("unsigned", {})) frozen_dict = freeze(event_dict) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 9579b1fe8b..d4cb602ebb 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index b9fb29be01..6bbba8d6ba 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 4849d3ce41..4f4914467c 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/validator.py b/synapse/events/validator.py index ebc6c30e62..0ee6872d1e 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py index 0112588656..7517c529d4 100644 --- a/synapse/federation/__init__.py +++ b/synapse/federation/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 73dc844d59..85c82a4623 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 8abf67b1b5..a4c29b484b 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -730,6 +730,7 @@ class _TransactionQueue(object): destinations = set(destinations) destinations.discard(self.server_name) + destinations.discard("localhost") logger.debug("Sending to: %s", str(destinations)) @@ -814,6 +815,8 @@ class _TransactionQueue(object): else: logger.info("TX [%s] is ready for retry", destination) + logger.info("TX [%s] _attempt_new_transaction", destination) + if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. @@ -826,6 +829,9 @@ class _TransactionQueue(object): pending_edus = self.pending_edus_by_dest.pop(destination, []) pending_failures = self.pending_failures_by_dest.pop(destination, []) + if pending_pdus: + logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", destination, len(pending_pdus)) + if not pending_pdus and not pending_edus and not pending_failures: return diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 0f11c6d491..1f0f06e0fe 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 1bcd0548c2..816f55bf39 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index d5df3c630b..fe071a4bc2 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 97ebd98917..38af034b4d 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 687b343a1d..1c9e7152c7 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index a907a66e12..91fceda2ac 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 02202692d4..808219bd10 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e23c5c2195..d26975a88a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index c98ae2cfb5..d297d71c03 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 854b2c73c6..7195de98b5 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -263,7 +263,7 @@ class MessageHandler(BaseHandler): } if event.membership == Membership.INVITE: - d["inviter"] = event.user_id + d["inviter"] = event.sender rooms_ret.append(d) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index f3abfecdee..8aeed99274 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 3f11e2dcf4..7777d3cc94 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes, Membership from synapse.util.logcontext import PreserveLoggingContext from ._base import BaseHandler @@ -203,7 +203,7 @@ class ProfileHandler(BaseHandler): for j in joins: content = { - "membership": j.content["membership"], + "membership": Membership.JOIN, } yield self.distributor.fire( @@ -212,9 +212,9 @@ class ProfileHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler yield msg_handler.create_and_send_event({ - "type": j.type, + "type": EventTypes.Member, "room_id": j.room_id, - "state_key": j.state_key, + "state_key": user.to_string(), "content": content, - "sender": j.state_key + "sender": user.to_string() }, ratelimit=False) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 15d8716455..732652c228 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index deefc3c11e..59719a1fae 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -245,14 +245,12 @@ class RoomMemberHandler(BaseHandler): self.distributor.declare("user_left_room") @defer.inlineCallbacks - def get_room_members(self, room_id, membership=Membership.JOIN): + def get_room_members(self, room_id): hs = self.hs - memberships = yield self.store.get_room_members( - room_id=room_id, membership=membership - ) + users = yield self.store.get_users_in_room(room_id) - defer.returnValue([hs.parse_userid(m.user_id) for m in memberships]) + defer.returnValue([hs.parse_userid(u) for u in users]) @defer.inlineCallbacks def fetch_room_distributions_into(self, room_id, localusers=None, @@ -531,11 +529,10 @@ class RoomListHandler(BaseHandler): def get_public_room_list(self): chunk = yield self.store.get_rooms(is_public=True) for room in chunk: - joined_members = yield self.store.get_room_members( + joined_users = yield self.store.get_users_in_room( room_id=room["room_id"], - membership=Membership.JOIN ) - room["num_joined_members"] = len(joined_members) + room["num_joined_members"] = len(joined_users) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 7626b07280..ab698b36e1 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py new file mode 100644 index 0000000000..d761890863 --- /dev/null +++ b/synapse/http/agent_name.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse import __version__ + +AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii") diff --git a/synapse/http/client.py b/synapse/http/client.py index 048a428905..e5d4939e2d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ # limitations under the License. +from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError @@ -51,7 +52,8 @@ class SimpleHttpClient(object): "POST", uri.encode("ascii"), headers=Headers({ - "Content-Type": ["application/x-www-form-urlencoded"] + b"Content-Type": [b"application/x-www-form-urlencoded"], + b"User-Agent": AGENT_NAME, }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -86,6 +88,9 @@ class SimpleHttpClient(object): response = yield self.agent.request( "GET", uri.encode("ascii"), + headers=Headers({ + b"User-Agent": AGENT_NAME, + }) ) body = yield readBody(response) @@ -108,7 +113,8 @@ class CaptchaServerHttpClient(SimpleHttpClient): url.encode("ascii"), bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ - "Content-Type": ["application/x-www-form-urlencoded"] + b"Content-Type": [b"application/x-www-form-urlencoded"], + b"User-Agent": AGENT_NAME, }) ) diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 9c8888f565..4ae45f136d 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8f4db59c75..aa14782b0f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone +from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext @@ -71,6 +72,7 @@ class MatrixFederationHttpClient(object): requests. """ + def __init__(self, hs): self.hs = hs self.signing_key = hs.config.signing_key[0] @@ -83,7 +85,7 @@ class MatrixFederationHttpClient(object): query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ - headers_dict[b"User-Agent"] = [b"Synapse"] + headers_dict[b"User-Agent"] = [AGENT_NAME] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( diff --git a/synapse/http/server.py b/synapse/http/server.py index f33859cf76..8015a22edf 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +14,16 @@ # limitations under the License. -from syutil.jsonutil import ( - encode_canonical_json, encode_pretty_printed_json -) +from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) from synapse.util.logcontext import LoggingContext +from syutil.jsonutil import ( + encode_canonical_json, encode_pretty_printed_json +) + from twisted.internet import defer, reactor from twisted.web import server, resource from twisted.web.server import NOT_DONE_YET @@ -230,6 +232,8 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") + request.setHeader(b"Server", AGENT_NAME) + request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) if send_cors: request.setHeader("Access-Control-Allow-Origin", "*") diff --git a/synapse/http/server_key_resource.py b/synapse/http/server_key_resource.py index b30ecead27..4fc491dc82 100644 --- a/synapse/http/server_key_resource.py +++ b/synapse/http/server_key_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v0/content_repository.py b/synapse/media/v0/content_repository.py index ce5d3d153e..79ae0e3d74 100644 --- a/synapse/media/v0/content_repository.py +++ b/synapse/media/v0/content_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/__init__.py b/synapse/media/v1/__init__.py index e69de29bb2..619999d268 100644 --- a/synapse/media/v1/__init__.py +++ b/synapse/media/v1/__init__.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import PIL.Image + +# check for JPEG support. +try: + PIL.Image._getdecoder("rgb", "jpeg", None) +except IOError as e: + if str(e).startswith("decoder jpeg not available"): + raise Exception( + "FATAL: jpeg codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip install -I pillow'" + ) +except Exception: + # any other exception is fine + pass + + +# check for PNG support. +try: + PIL.Image._getdecoder("rgb", "zip", None) +except IOError as e: + if str(e).startswith("decoder zip not available"): + raise Exception( + "FATAL: zip codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip install -I pillow'" + ) +except Exception: + # any other exception is fine + pass diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 499be8cca0..688e7376ad 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -202,7 +202,8 @@ class BaseMediaResource(Resource): defer.returnValue(media_info) @defer.inlineCallbacks - def _respond_with_file(self, request, media_type, file_path): + def _respond_with_file(self, request, media_type, file_path, + file_size=None): logger.debug("Responding with %r", file_path) if os.path.isfile(file_path): @@ -216,13 +217,20 @@ class BaseMediaResource(Resource): request.setHeader( b"Cache-Control", b"public,max-age=86400,s-maxage=86400" ) + if file_size is None: + stat = os.stat(file_path) + file_size = stat.st_size + + request.setHeader( + b"Content-Length", b"%d" % (file_size,) + ) with open(file_path, "rb") as f: yield FileSender().beginFileTransfer(f, request) request.finish() else: - self._respond_404() + self._respond_404(request) def _get_thumbnail_requirements(self, media_type): if media_type == "image/jpeg": diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py index f3a6804e05..c585bb11f7 100644 --- a/synapse/media/v1/download_resource.py +++ b/synapse/media/v1/download_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -46,23 +46,29 @@ class DownloadResource(BaseMediaResource): def _respond_local_file(self, request, media_id): media_info = yield self.store.get_local_media(media_id) if not media_info: - self._respond_404() + self._respond_404(request) return media_type = media_info["media_type"] + media_length = media_info["media_length"] file_path = self.filepaths.local_media_filepath(media_id) - yield self._respond_with_file(request, media_type, file_path) + yield self._respond_with_file( + request, media_type, file_path, media_length + ) @defer.inlineCallbacks def _respond_remote_file(self, request, server_name, media_id): media_info = yield self._get_remote_media(server_name, media_id) media_type = media_info["media_type"] + media_length = media_info["media_length"] filesystem_id = media_info["filesystem_id"] file_path = self.filepaths.remote_media_filepath( server_name, filesystem_id ) - yield self._respond_with_file(request, media_type, file_path) + yield self._respond_with_file( + request, media_type, file_path, media_length + ) diff --git a/synapse/media/v1/filepath.py b/synapse/media/v1/filepath.py index 0078bc3d40..ed9a58e9d9 100644 --- a/synapse/media/v1/filepath.py +++ b/synapse/media/v1/filepath.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index 2070ec3c7f..461cc001f1 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py index 5ddcf54b9f..84f5e3463c 100644 --- a/synapse/media/v1/thumbnail_resource.py +++ b/synapse/media/v1/thumbnail_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -100,11 +100,12 @@ class ThumbnailResource(BaseMediaResource): t_type = thumbnail_info["thumbnail_type"] t_method = thumbnail_info["thumbnail_method"] file_id = thumbnail_info["filesystem_id"] + t_length = thumbnail_info["thumbnail_length"] file_path = self.filepaths.remote_media_thumbnail( server_name, file_id, t_width, t_height, t_type, t_method, ) - yield self._respond_with_file(request, t_type, file_path) + yield self._respond_with_file(request, t_type, file_path, t_length) else: yield self._respond_default_thumbnail( request, media_info, width, height, method, m_type, @@ -139,11 +140,12 @@ class ThumbnailResource(BaseMediaResource): t_height = thumbnail_info["thumbnail_height"] t_type = thumbnail_info["thumbnail_type"] t_method = thumbnail_info["thumbnail_method"] + t_length = thumbnail_info["thumbnail_length"] file_path = self.filepaths.default_thumbnail( top_level_type, sub_type, t_width, t_height, t_type, t_method, ) - yield self.respond_with_file(request, t_type, file_path) + yield self.respond_with_file(request, t_type, file_path, t_length) def _select_thumbnail(self, desired_width, desired_height, desired_method, desired_type, thumbnail_infos): diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py index 774ae4538f..bc86efea8f 100644 --- a/synapse/media/v1/thumbnailer.py +++ b/synapse/media/v1/thumbnailer.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ class Thumbnailer(object): def scale(self, output_path, width, height, output_type): """Rescales the image to the given dimensions""" - scaled = self.image.resize((width, height), Image.BILINEAR) + scaled = self.image.resize((width, height), Image.ANTIALIAS) return self.save_image(scaled, output_type, output_path) def crop(self, output_path, width, height, output_type): @@ -65,7 +65,7 @@ class Thumbnailer(object): if width * self.height > height * self.width: scaled_height = (width * self.height) // self.width scaled_image = self.image.resize( - (width, scaled_height), Image.BILINEAR + (width, scaled_height), Image.ANTIALIAS ) crop_top = (scaled_height - height) // 2 crop_bottom = height + crop_top @@ -73,7 +73,7 @@ class Thumbnailer(object): else: scaled_width = (height * self.width) // self.height scaled_image = self.image.resize( - (scaled_width, height), Image.BILINEAR + (scaled_width, height), Image.ANTIALIAS ) crop_left = (scaled_width - width) // 2 crop_right = width + crop_left diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index 5645b0df46..b1718a630b 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/notifier.py b/synapse/notifier.py index 383230caf1..b9d52d0c4c 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index a59630ec96..88ec9cd27d 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/admin.py b/synapse/rest/admin.py index d74c551512..0aa83514c8 100644 --- a/synapse/rest/admin.py +++ b/synapse/rest/admin.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/base.py b/synapse/rest/base.py index 06eda2587c..c583945527 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 868fa7abed..7ff44fdd9e 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/events.py b/synapse/rest/events.py index 3c1b041bfe..cf6d13f817 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/initial_sync.py b/synapse/rest/initial_sync.py index a1cb442256..a571589581 100644 --- a/synapse/rest/initial_sync.py +++ b/synapse/rest/initial_sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/login.py b/synapse/rest/login.py index 875da076af..6b8deff67b 100644 --- a/synapse/rest/login.py +++ b/synapse/rest/login.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index 9b42131628..ca4d2d21f0 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py index 351863ab5d..dc6eb424b0 100644 --- a/synapse/rest/profile.py +++ b/synapse/rest/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 4f0f5a7531..e3b26902d9 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 005a9f6f44..e40773758a 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/transactions.py b/synapse/rest/transactions.py index 31377bd41d..d933fea18a 100644 --- a/synapse/rest/transactions.py +++ b/synapse/rest/transactions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/voip.py b/synapse/rest/voip.py index 432c2475f8..011c35e69b 100644 --- a/synapse/rest/voip.py +++ b/synapse/rest/voip.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/server.py b/synapse/server.py index e4021481e8..c3bf46abbf 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/state.py b/synapse/state.py index 15397e2f38..8144fa02b4 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e6bb665932..4beb951b9f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -66,7 +66,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 10 +SCHEMA_VERSION = 11 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e0d97f440b..f660fc6eaf 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -64,7 +64,7 @@ class LoggingTransaction(object): # Don't let logging failures stop SQL from working pass - start = time.clock() * 1000 + start = time.time() * 1000 try: return self.txn.execute( sql, *args, **kwargs @@ -73,7 +73,7 @@ class LoggingTransaction(object): logger.exception("[SQL FAIL] {%s}", self.name) raise finally: - end = time.clock() * 1000 + end = time.time() * 1000 sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) @@ -93,7 +93,7 @@ class SQLBaseStore(object): def inner_func(txn, *args, **kwargs): with LoggingContext("runInteraction") as context: current_context.copy_to(context) - start = time.clock() * 1000 + start = time.time() * 1000 txn_id = SQLBaseStore._TXN_ID # We don't really need these to be unique, so lets stop it from @@ -109,7 +109,7 @@ class SQLBaseStore(object): logger.exception("[TXN FAIL] {%s}", name) raise finally: - end = time.clock() * 1000 + end = time.time() * 1000 transaction_logger.debug( "[TXN END] {%s} %f", name, end - start @@ -434,23 +434,31 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) - def _get_events(self, event_ids): + def _get_events(self, event_ids, check_redacted=True, + get_prev_content=False): return self.runInteraction( - "_get_events", self._get_events_txn, event_ids + "_get_events", self._get_events_txn, event_ids, + check_redacted=check_redacted, get_prev_content=get_prev_content, ) - def _get_events_txn(self, txn, event_ids): - events = [] - for e_id in event_ids: - ev = self._get_event_txn(txn, e_id) + def _get_events_txn(self, txn, event_ids, check_redacted=True, + get_prev_content=False): + if not event_ids: + return [] - if ev: - events.append(ev) + events = [ + self._get_event_txn( + txn, event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content + ) + for event_id in event_ids + ] - return events + return [e for e in events if e] def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=True): + get_prev_content=False): sql = ( "SELECT internal_metadata, json, r.event_id FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " @@ -467,6 +475,14 @@ class SQLBaseStore(object): internal_metadata, js, redacted = res + return self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + + def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, + check_redacted=True, get_prev_content=False): d = json.loads(js) internal_metadata = json.loads(internal_metadata) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 2be9c41374..68b7d59693 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index fb2eb21713..0cbcdd1b55 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py index 21511577c5..fcf011b234 100644 --- a/synapse/storage/feedback.py +++ b/synapse/storage/feedback.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index fd705138e6..1f244019fc 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index 18c068d3d9..7101d2beec 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 71b2bb084d..1dcd34723b 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 7e1fdd9d88..153c7ad027 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 4d15005c9e..75dffa4db2 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/room.py b/synapse/storage/room.py index c20abbfe4d..978b2c4a48 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 05b275663e..e59e65529b 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ from twisted.internet import defer +from collections import namedtuple + from ._base import SQLBaseStore from synapse.api.constants import Membership @@ -24,6 +26,12 @@ import logging logger = logging.getLogger(__name__) +RoomsForUser = namedtuple( + "RoomsForUser", + ("room_id", "sender", "membership") +) + + class RoomMemberStore(SQLBaseStore): def _store_room_member_txn(self, txn, event): @@ -123,6 +131,19 @@ class RoomMemberStore(SQLBaseStore): else: return None + def get_users_in_room(self, room_id): + def f(txn): + sql = ( + "SELECT m.user_id FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE m.membership = ? AND m.room_id = ?" + ) + + txn.execute(sql, (Membership.JOIN, room_id)) + return [r[0] for r in txn.fetchall()] + return self.runInteraction("get_users_in_room", f) + def get_room_members(self, room_id, membership=None): """Retrieve the current room member list for a room. @@ -150,19 +171,37 @@ class RoomMemberStore(SQLBaseStore): membership_list (list): A list of synapse.api.constants.Membership values which the user must be in. Returns: - A list of RoomMemberEvent objects + A list of dictionary objects, with room_id, membership and sender + defined. """ if not membership_list: return defer.succeed(None) - args = [user_id] - args.extend(membership_list) - where_clause = "user_id = ? AND (%s)" % ( " OR ".join(["membership = ?" for _ in membership_list]), ) - return self._get_members_query(where_clause, args) + args = [user_id] + args.extend(membership_list) + + def f(txn): + sql = ( + "SELECT m.room_id, m.sender, m.membership" + " FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE %s" + ) % (where_clause,) + + txn.execute(sql, args) + return [ + RoomsForUser(**r) for r in self.cursor_to_dict(txn) + ] + + return self.runInteraction( + "get_rooms_for_user_where_membership_is", + f + ) def get_joined_hosts_for_room(self, room_id): return self._simple_select_onecol( @@ -183,20 +222,14 @@ class RoomMemberStore(SQLBaseStore): ) def _get_members_query_txn(self, txn, where_clause, where_values): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "SELECT e.* FROM events as e " "INNER JOIN room_memberships as m " "ON e.event_id = m.event_id " "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " "WHERE %(where)s " ) % { - "redacted": del_sql, "where": where_clause, } @@ -206,26 +239,28 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results - @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a room. """ - user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) - sql = ( - "SELECT m.room_id FROM room_memberships as m " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE m.membership = 'join' " - "AND (%(clause)s) " - # TODO(paul): We've got duplicate rows in the database somewhere - # so we have to DISTINCT m.user_id here - "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" - ) % {"clause": user_list_clause} + def interaction(txn): + user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) + sql = ( + "SELECT m.room_id FROM room_memberships as m " + "INNER JOIN current_state_events as c " + "ON m.event_id = c.event_id " + "WHERE m.membership = 'join' " + "AND (%(clause)s) " + # TODO(paul): We've got duplicate rows in the database somewhere + # so we have to DISTINCT m.user_id here + "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" + ) % {"clause": user_list_clause} + + args = list(user_id_list) + args.append(len(user_id_list)) - args = list(user_id_list) - args.append(len(user_id_list)) + txn.execute(sql, args) - rows = yield self._execute(None, sql, *args) + return len(txn.fetchall()) > 0 - defer.returnValue(len(rows) > 0) + return self.runInteraction("user_rooms_intersect", interaction) diff --git a/synapse/storage/schema/delta/v11.sql b/synapse/storage/schema/delta/v11.sql new file mode 100644 index 0000000000..313592221b --- /dev/null +++ b/synapse/storage/schema/delta/v11.sql @@ -0,0 +1,16 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); \ No newline at end of file diff --git a/synapse/storage/schema/delta/v2.sql b/synapse/storage/schema/delta/v2.sql index 73b140465e..f740f6dd5d 100644 --- a/synapse/storage/schema/delta/v2.sql +++ b/synapse/storage/schema/delta/v2.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v3.sql b/synapse/storage/schema/delta/v3.sql index cade295989..c67e38ff52 100644 --- a/synapse/storage/schema/delta/v3.sql +++ b/synapse/storage/schema/delta/v3.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v4.sql b/synapse/storage/schema/delta/v4.sql index 25d2ead450..d3807b7686 100644 --- a/synapse/storage/schema/delta/v4.sql +++ b/synapse/storage/schema/delta/v4.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS redactions ( event_id TEXT NOT NULL, redacts TEXT NOT NULL, diff --git a/synapse/storage/schema/delta/v5.sql b/synapse/storage/schema/delta/v5.sql index af9df11aa9..0874a15431 100644 --- a/synapse/storage/schema/delta/v5.sql +++ b/synapse/storage/schema/delta/v5.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS user_ips ( user TEXT NOT NULL, diff --git a/synapse/storage/schema/delta/v6.sql b/synapse/storage/schema/delta/v6.sql index 9bf2068d84..a9e0a4fe0d 100644 --- a/synapse/storage/schema/delta/v6.sql +++ b/synapse/storage/schema/delta/v6.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v8.sql b/synapse/storage/schema/delta/v8.sql index daf6646ed5..1e9f8b18cb 100644 --- a/synapse/storage/schema/delta/v8.sql +++ b/synapse/storage/schema/delta/v8.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql index 0af29733a0..455d51a70c 100644 --- a/synapse/storage/schema/delta/v9.sql +++ b/synapse/storage/schema/delta/v9.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index be1c72a775..1e766d6db2 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id TEXT NOT NULL, diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/event_signatures.sql index b6b56b47a2..c28c39c48a 100644 --- a/synapse/storage/schema/event_signatures.sql +++ b/synapse/storage/schema/event_signatures.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 253f9f779b..dd00c1cd2f 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/keys.sql index 9bf2068d84..a9e0a4fe0d 100644 --- a/synapse/storage/schema/keys.sql +++ b/synapse/storage/schema/keys.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/media_repository.sql b/synapse/storage/schema/media_repository.sql index b785fa0208..afdf48cbfb 100644 --- a/synapse/storage/schema/media_repository.sql +++ b/synapse/storage/schema/media_repository.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/presence.sql b/synapse/storage/schema/presence.sql index 595b3b5a69..f9f8db9697 100644 --- a/synapse/storage/schema/presence.sql +++ b/synapse/storage/schema/presence.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/profiles.sql b/synapse/storage/schema/profiles.sql index 58209f1af0..f06a528b4d 100644 --- a/synapse/storage/schema/profiles.sql +++ b/synapse/storage/schema/profiles.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/redactions.sql b/synapse/storage/schema/redactions.sql index 4c2829d05d..5011d95db8 100644 --- a/synapse/storage/schema/redactions.sql +++ b/synapse/storage/schema/redactions.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS redactions ( event_id TEXT NOT NULL, redacts TEXT NOT NULL, diff --git a/synapse/storage/schema/room_aliases.sql b/synapse/storage/schema/room_aliases.sql index 9191016814..0d2df01603 100644 --- a/synapse/storage/schema/room_aliases.sql +++ b/synapse/storage/schema/room_aliases.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql index 2c48d6daca..1fe8f1e430 100644 --- a/synapse/storage/schema/state.sql +++ b/synapse/storage/schema/state.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index de461bfa15..2d30f99b06 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,7 @@ CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destinatio CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( destination ); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); diff --git a/synapse/storage/schema/users.sql b/synapse/storage/schema/users.sql index 8244f733bd..08ccfdac0a 100644 --- a/synapse/storage/schema/users.sql +++ b/synapse/storage/schema/users.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 3a705119fd..d0d53770f2 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/state.py b/synapse/storage/state.py index afe3e5edea..5327517704 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ from ._base import SQLBaseStore +import logging + +logger = logging.getLogger(__name__) + class StateStore(SQLBaseStore): """ Keeps track of the state at a given event. @@ -62,14 +66,8 @@ class StateStore(SQLBaseStore): keyvalues={"state_group": group}, retcol="event_id", ) - state = [] - for state_id in state_ids: - s = self._get_events_txn( - txn, - [state_id], - ) - if s: - state.extend(s) + + state = self._get_events_txn(txn, state_ids) res[group] = state diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3405cb365e..bedc3c6c52 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -137,7 +137,6 @@ class StreamStore(SQLBaseStore): with_feedback=with_feedback, ) - @defer.inlineCallbacks @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): @@ -157,11 +156,6 @@ class StreamStore(SQLBaseStore): "WHERE m.user_id = ? " ) - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - if limit: limit = max(limit, MAX_STREAM_SIZE) else: @@ -172,38 +166,42 @@ class StreamStore(SQLBaseStore): to_id = _parse_stream_token(to_key) if from_key == to_key: - defer.returnValue(([], to_key)) - return + return defer.succeed(([], to_key)) sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events AS e WHERE " + "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { - "redacted": del_sql, "current": current_room_membership_sql, "invites": membership_sql, "limit": limit } - rows = yield self._execute_and_decode( - sql, - user_id, user_id, from_id, to_id - ) + def f(txn): + txn.execute(sql, (user_id, user_id, from_id, to_id,)) - ret = yield self._parse_events(rows) + rows = self.cursor_to_dict(txn) - if rows: - key = "s%d" % max([r["stream_ordering"] for r in rows]) - else: - # Assume we didn't get anything because there was nothing to get. - key = to_key + ret = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) + + if rows: + key = "s%d" % max([r["stream_ordering"] for r in rows]) + else: + # Assume we didn't get anything because there was nothing to + # get. + key = to_key + + return ret, key - defer.returnValue((ret, key)) + return self.runInteraction("get_room_events_stream", f) - @defer.inlineCallbacks @log_function def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, @@ -221,7 +219,9 @@ class StreamStore(SQLBaseStore): bounds = _get_token_bound(from_key, from_comp) if to_key: - bounds = "%s AND %s" % (bounds, _get_token_bound(to_key, to_comp)) + bounds = "%s AND %s" % ( + bounds, _get_token_bound(to_key, to_comp) + ) if int(limit) > 0: args.append(int(limit)) @@ -229,87 +229,78 @@ class StreamStore(SQLBaseStore): else: limit_str = "" - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = events.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events" + "SELECT * FROM events" " WHERE outlier = 0 AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" ) % { - "redacted": del_sql, "bounds": bounds, "order": order, "limit": limit_str } - rows = yield self._execute_and_decode( - sql, - *args - ) - - if rows: - topo = rows[-1]["topological_ordering"] - toke = rows[-1]["stream_ordering"] - if direction == 'b': - topo -= 1 - toke -= 1 - next_token = "t%s-%s" % (topo, toke) - else: - # TODO (erikj): We should work out what to do here instead. - next_token = to_key if to_key else from_key + def f(txn): + txn.execute(sql, args) + + rows = self.cursor_to_dict(txn) + + if rows: + topo = rows[-1]["topological_ordering"] + toke = rows[-1]["stream_ordering"] + if direction == 'b': + topo -= 1 + toke -= 1 + next_token = "t%s-%s" % (topo, toke) + else: + # TODO (erikj): We should work out what to do here instead. + next_token = to_key if to_key else from_key + + events = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) - events = yield self._parse_events(rows) + return events, next_token, - defer.returnValue( - ( - events, - next_token - ) - ) + return self.runInteraction("paginate_room_events", f) - @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, end_token, with_feedback=False): # TODO (erikj): Handle compressed feedback - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = events.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events " + "SELECT stream_ordering, topological_ordering, event_id FROM events " "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) % { - "redacted": del_sql, - } - - rows = yield self._execute_and_decode( - sql, - room_id, end_token, limit ) - rows.reverse() # As we selected with reverse ordering + def f(txn): + txn.execute(sql, (room_id, end_token, limit,)) - if rows: - topo = rows[0]["topological_ordering"] - toke = rows[0]["stream_ordering"] - start_token = "t%s-%s" % (topo, toke) + rows = self.cursor_to_dict(txn) - token = (start_token, end_token) - else: - token = (end_token, end_token) + rows.reverse() # As we selected with reverse ordering - events = yield self._parse_events(rows) + if rows: + topo = rows[0]["topological_ordering"] + toke = rows[0]["stream_ordering"] + start_token = "t%s-%s" % (topo, toke) + + token = (start_token, end_token) + else: + token = (end_token, end_token) + + events = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) - ret = (events, token) + return events, token - defer.returnValue(ret) + return self.runInteraction("get_recent_events_for_room", f) def get_room_events_max_id(self): return self.runInteraction( diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 423cc3f02a..e06ef35690 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/__init__.py b/synapse/streams/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/streams/__init__.py +++ b/synapse/streams/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/config.py b/synapse/streams/config.py index 2114c940e7..2ec7c5403b 100644 --- a/synapse/streams/config.py +++ b/synapse/streams/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/events.py b/synapse/streams/events.py index fb698d2d71..5c8e54b78b 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/types.py b/synapse/types.py index 7c533193e1..faac729ff2 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 7ec5033ceb..4e837a918e 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/async.py b/synapse/util/async.py index 7dd3ec3a72..c4fe5d522f 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 6925ac96b6..9d9c350397 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/emailutils.py b/synapse/util/emailutils.py index 7038cab6c2..7f9a77bf44 100644 --- a/synapse/util/emailutils.py +++ b/synapse/util/emailutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index fcfb38b3b4..a13a2015e4 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,10 +17,11 @@ from frozendict import frozendict def freeze(o): - if isinstance(o, dict) or isinstance(o, frozendict): + t = type(o) + if t is dict: return frozendict({k: freeze(v) for k, v in o.items()}) - if isinstance(o, basestring): + if t is str or t is unicode: return o try: diff --git a/synapse/util/jsonobject.py b/synapse/util/jsonobject.py index e79b68f661..0765f7d217 100644 --- a/synapse/util/jsonobject.py +++ b/synapse/util/jsonobject.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/lockutils.py b/synapse/util/lockutils.py index 3a84c09db4..33edc5c20e 100644 --- a/synapse/util/lockutils.py +++ b/synapse/util/lockutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 7d85018d97..da7872e95d 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -1,3 +1,17 @@ +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import threading import logging diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index 903a6cf1b3..fd9ac4d4d4 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,14 +19,37 @@ from functools import wraps import logging import inspect +import time + + +_TIME_FUNC_ID = 0 + + +def _log_debug_as_f(f, msg, msg_args): + name = f.__module__ + logger = logging.getLogger(name) + + if logger.isEnabledFor(logging.DEBUG): + lineno = f.func_code.co_firstlineno + pathname = f.func_code.co_filename + + record = logging.LogRecord( + name=name, + level=logging.DEBUG, + pathname=pathname, + lineno=lineno, + msg=msg, + args=msg_args, + exc_info=None + ) + + logger.handle(record) def log_function(f): """ Function decorator that logs every call to that function. """ func_name = f.__name__ - lineno = f.func_code.co_firstlineno - pathname = f.func_code.co_filename @wraps(f) def wrapped(*args, **kwargs): @@ -52,24 +75,50 @@ def log_function(f): "args": ", ".join(func_args) } - record = logging.LogRecord( - name=name, - level=level, - pathname=pathname, - lineno=lineno, - msg="Invoked '%(func_name)s' with args: %(args)s", - args=msg_args, - exc_info=None + _log_debug_as_f( + f, + "Invoked '%(func_name)s' with args: %(args)s", + msg_args ) - logger.handle(record) - return f(*args, **kwargs) wrapped.__name__ = func_name return wrapped +def time_function(f): + func_name = f.__name__ + + @wraps(f) + def wrapped(*args, **kwargs): + global _TIME_FUNC_ID + id = _TIME_FUNC_ID + _TIME_FUNC_ID += 1 + + start = time.clock() * 1000 + + try: + _log_debug_as_f( + f, + "[FUNC START] {%s-%d}", + (func_name, id), + ) + + r = f(*args, **kwargs) + finally: + end = time.clock() * 1000 + _log_debug_as_f( + f, + "[FUNC END] {%s-%d} %f", + (func_name, id, end-start,), + ) + + return r + + return wrapped + + def trace_function(f): func_name = f.__name__ linenum = f.func_code.co_firstlineno diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index 8767e437dd..ea53a8085c 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. |