diff options
| author | Erik Johnston <erik@matrix.org> | 2015-01-07 14:34:00 +0000 |
|---|---|---|
| committer | Erik Johnston <erik@matrix.org> | 2015-01-07 14:34:00 +0000 |
| commit | bacaa215eb2e1c11e50bbc3fb0e43e28a463458b (patch) | |
| tree | 27510a6358b8bc219ccd456622b963a3b394bee3 | |
| parent | Merge branch 'hotfixes-v0.6.0a' (diff) | |
| parent | Improve change log (diff) | |
| download | synapse-bacaa215eb2e1c11e50bbc3fb0e43e28a463458b.tar.xz | |
Merge branch 'release-v0.6.1' of github.com:matrix-org/synapse v0.6.1
137 files changed, 601 insertions, 310 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index 813ad364ea..297ae914fd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +Changes in synapse 0.6.1 (2015-01-07) +===================================== + + * Major optimizations to improve performance of initial sync and event sending + in large rooms (by up to 10x) + * Media repository now includes a Content-Length header on media downloads. + * Improve quality of thumbnails by changing resizing algorithm. + Changes in synapse 0.6.0 (2014-12-16) ===================================== diff --git a/README.rst b/README.rst index f5d2b0af38..92b94bcd7d 100644 --- a/README.rst +++ b/README.rst @@ -108,6 +108,9 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into ``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX. +For reliable VoIP calls to be routed via this homeserver, you MUST configure +a TURN server. See docs/turn-howto.rst for details. + Troubleshooting Installation ---------------------------- @@ -239,6 +242,11 @@ Upgrading an existing homeserver IMPORTANT: Before upgrading an existing homeserver to a new version, please refer to UPGRADE.rst for any additional instructions. +Otherwise, simply re-install the new codebase over the current one - e.g. +by ``pip install --user --process-dependency-links +https://github.com/matrix-org/synapse/tarball/master`` +if using pip, or by ``git pull`` if running off a git working copy. + Setting up Federation ===================== diff --git a/VERSION b/VERSION index a918a2aa18..ee6cdce3c2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.0 +0.6.1 diff --git a/docs/media_repository.rst b/docs/media_repository.rst index e4a6974041..1037b5be63 100644 --- a/docs/media_repository.rst +++ b/docs/media_repository.rst @@ -1,6 +1,8 @@ -Media Repository +Media Repository ================ +*Synapse implementation-specific details for the media repository* + The media repository is where attachments and avatar photos are stored. It stores attachment content and thumbnails for media uploaded by local users. It caches attachment content and thumbnails for media uploaded by remote users. diff --git a/scripts/copyrighter.pl b/scripts/copyrighter.pl index 7c03ef21fc..a913d74c8d 100755 --- a/scripts/copyrighter.pl +++ b/scripts/copyrighter.pl @@ -14,7 +14,7 @@ # limitations under the License. $copyright = <<EOT; -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/__init__.py b/synapse/__init__.py index 1cef40925f..c3f1ac63be 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.0" +__version__ = "0.6.1" diff --git a/synapse/api/__init__.py b/synapse/api/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/api/__init__.py +++ b/synapse/api/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 70245aba04..e31482cfaa 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 4fc8b79a40..7ee6dcc46e 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e250b9b211..2b049debf3 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index b25358090f..3f9ad4ce89 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/urls.py b/synapse/api/urls.py index d7625127f8..a299392049 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index c78b6f574d..5fec8da7ca 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 52a0b729f4..363c20f994 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/__init__.py b/synapse/config/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/config/__init__.py +++ b/synapse/config/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 1cdd03e414..dfc115d8e8 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 4ed9070b9e..7e21c7414d 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/database.py b/synapse/config/database.py index 0aac8c8382..0d33583a7d 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/email.py b/synapse/config/email.py index 6bab133224..f0854f8c37 100644 --- a/synapse/config/email.py +++ b/synapse/config/email.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 5a11fd6c76..b0fe217459 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/logger.py b/synapse/config/logger.py index d352ea9be9..15383b3184 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index fb63ed7d9b..17c7e64ce7 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/repository.py b/synapse/config/repository.py index f1b7b1b74e..e1827f05e4 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014, 2015 matrix.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ class ContentRepositoryConfig(Config): super(ContentRepositoryConfig, cls).add_arguments(parser) db_group = parser.add_argument_group("content_repository") db_group.add_argument( - "--max-upload-size", default="1M" + "--max-upload-size", default="10M" ) db_group.add_argument( "--media-store-path", default=cls.default_path("media_store") diff --git a/synapse/config/server.py b/synapse/config/server.py index f8a0844b8c..4f73c85466 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 3600c3ea9e..384b29e7ba 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/voip.py b/synapse/config/voip.py index 06675966ce..a2b822719f 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/__init__.py b/synapse/crypto/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/crypto/__init__.py +++ b/synapse/crypto/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 3143322d9c..24d4abf3e9 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 21c19c971d..6633b19565 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 3f37c99261..9c910fa3fc 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index ceb03ce6c2..3fb99f7125 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 34b1b944ab..4252e5ab5c 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,12 +15,10 @@ from synapse.util.frozenutils import freeze, unfreeze -import copy - class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): - self.__dict__ = copy.deepcopy(internal_metadata_dict) + self.__dict__ = internal_metadata_dict def get_dict(self): return dict(self.__dict__) @@ -49,10 +47,10 @@ def _event_dict_property(key): class EventBase(object): def __init__(self, event_dict, signatures={}, unsigned={}, internal_metadata_dict={}): - self.signatures = copy.deepcopy(signatures) - self.unsigned = copy.deepcopy(unsigned) + self.signatures = signatures + self.unsigned = unsigned - self._event_dict = copy.deepcopy(event_dict) + self._event_dict = event_dict self.internal_metadata = _EventInternalMetadata( internal_metadata_dict @@ -112,10 +110,16 @@ class EventBase(object): class FrozenEvent(EventBase): def __init__(self, event_dict, internal_metadata_dict={}): - event_dict = copy.deepcopy(event_dict) + event_dict = dict(event_dict) + + # Signatures is a dict of dicts, and this is faster than doing a + # copy.deepcopy + signatures = { + name: {sig_id: sig for sig_id, sig in sigs.items()} + for name, sigs in event_dict.pop("signatures", {}).items() + } - signatures = copy.deepcopy(event_dict.pop("signatures", {})) - unsigned = copy.deepcopy(event_dict.pop("unsigned", {})) + unsigned = dict(event_dict.pop("unsigned", {})) frozen_dict = freeze(event_dict) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 9579b1fe8b..d4cb602ebb 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index b9fb29be01..6bbba8d6ba 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 4849d3ce41..4f4914467c 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/validator.py b/synapse/events/validator.py index ebc6c30e62..0ee6872d1e 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py index 0112588656..7517c529d4 100644 --- a/synapse/federation/__init__.py +++ b/synapse/federation/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 73dc844d59..85c82a4623 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 8abf67b1b5..a4c29b484b 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -730,6 +730,7 @@ class _TransactionQueue(object): destinations = set(destinations) destinations.discard(self.server_name) + destinations.discard("localhost") logger.debug("Sending to: %s", str(destinations)) @@ -814,6 +815,8 @@ class _TransactionQueue(object): else: logger.info("TX [%s] is ready for retry", destination) + logger.info("TX [%s] _attempt_new_transaction", destination) + if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. @@ -826,6 +829,9 @@ class _TransactionQueue(object): pending_edus = self.pending_edus_by_dest.pop(destination, []) pending_failures = self.pending_failures_by_dest.pop(destination, []) + if pending_pdus: + logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", destination, len(pending_pdus)) + if not pending_pdus and not pending_edus and not pending_failures: return diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 0f11c6d491..1f0f06e0fe 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 1bcd0548c2..816f55bf39 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index d5df3c630b..fe071a4bc2 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 97ebd98917..38af034b4d 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 687b343a1d..1c9e7152c7 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index a907a66e12..91fceda2ac 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 02202692d4..808219bd10 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e23c5c2195..d26975a88a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index c98ae2cfb5..d297d71c03 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 854b2c73c6..7195de98b5 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -263,7 +263,7 @@ class MessageHandler(BaseHandler): } if event.membership == Membership.INVITE: - d["inviter"] = event.user_id + d["inviter"] = event.sender rooms_ret.append(d) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index f3abfecdee..8aeed99274 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 3f11e2dcf4..7777d3cc94 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes, Membership from synapse.util.logcontext import PreserveLoggingContext from ._base import BaseHandler @@ -203,7 +203,7 @@ class ProfileHandler(BaseHandler): for j in joins: content = { - "membership": j.content["membership"], + "membership": Membership.JOIN, } yield self.distributor.fire( @@ -212,9 +212,9 @@ class ProfileHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler yield msg_handler.create_and_send_event({ - "type": j.type, + "type": EventTypes.Member, "room_id": j.room_id, - "state_key": j.state_key, + "state_key": user.to_string(), "content": content, - "sender": j.state_key + "sender": user.to_string() }, ratelimit=False) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 15d8716455..732652c228 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index deefc3c11e..59719a1fae 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -245,14 +245,12 @@ class RoomMemberHandler(BaseHandler): self.distributor.declare("user_left_room") @defer.inlineCallbacks - def get_room_members(self, room_id, membership=Membership.JOIN): + def get_room_members(self, room_id): hs = self.hs - memberships = yield self.store.get_room_members( - room_id=room_id, membership=membership - ) + users = yield self.store.get_users_in_room(room_id) - defer.returnValue([hs.parse_userid(m.user_id) for m in memberships]) + defer.returnValue([hs.parse_userid(u) for u in users]) @defer.inlineCallbacks def fetch_room_distributions_into(self, room_id, localusers=None, @@ -531,11 +529,10 @@ class RoomListHandler(BaseHandler): def get_public_room_list(self): chunk = yield self.store.get_rooms(is_public=True) for room in chunk: - joined_members = yield self.store.get_room_members( + joined_users = yield self.store.get_users_in_room( room_id=room["room_id"], - membership=Membership.JOIN ) - room["num_joined_members"] = len(joined_members) + room["num_joined_members"] = len(joined_users) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 7626b07280..ab698b36e1 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py new file mode 100644 index 0000000000..d761890863 --- /dev/null +++ b/synapse/http/agent_name.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse import __version__ + +AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii") diff --git a/synapse/http/client.py b/synapse/http/client.py index 048a428905..e5d4939e2d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ # limitations under the License. +from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError @@ -51,7 +52,8 @@ class SimpleHttpClient(object): "POST", uri.encode("ascii"), headers=Headers({ - "Content-Type": ["application/x-www-form-urlencoded"] + b"Content-Type": [b"application/x-www-form-urlencoded"], + b"User-Agent": AGENT_NAME, }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -86,6 +88,9 @@ class SimpleHttpClient(object): response = yield self.agent.request( "GET", uri.encode("ascii"), + headers=Headers({ + b"User-Agent": AGENT_NAME, + }) ) body = yield readBody(response) @@ -108,7 +113,8 @@ class CaptchaServerHttpClient(SimpleHttpClient): url.encode("ascii"), bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ - "Content-Type": ["application/x-www-form-urlencoded"] + b"Content-Type": [b"application/x-www-form-urlencoded"], + b"User-Agent": AGENT_NAME, }) ) diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 9c8888f565..4ae45f136d 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8f4db59c75..aa14782b0f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone +from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext @@ -71,6 +72,7 @@ class MatrixFederationHttpClient(object): requests. """ + def __init__(self, hs): self.hs = hs self.signing_key = hs.config.signing_key[0] @@ -83,7 +85,7 @@ class MatrixFederationHttpClient(object): query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ - headers_dict[b"User-Agent"] = [b"Synapse"] + headers_dict[b"User-Agent"] = [AGENT_NAME] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( diff --git a/synapse/http/server.py b/synapse/http/server.py index f33859cf76..8015a22edf 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +14,16 @@ # limitations under the License. -from syutil.jsonutil import ( - encode_canonical_json, encode_pretty_printed_json -) +from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) from synapse.util.logcontext import LoggingContext +from syutil.jsonutil import ( + encode_canonical_json, encode_pretty_printed_json +) + from twisted.internet import defer, reactor from twisted.web import server, resource from twisted.web.server import NOT_DONE_YET @@ -230,6 +232,8 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") + request.setHeader(b"Server", AGENT_NAME) + request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) if send_cors: request.setHeader("Access-Control-Allow-Origin", "*") diff --git a/synapse/http/server_key_resource.py b/synapse/http/server_key_resource.py index b30ecead27..4fc491dc82 100644 --- a/synapse/http/server_key_resource.py +++ b/synapse/http/server_key_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v0/content_repository.py b/synapse/media/v0/content_repository.py index ce5d3d153e..79ae0e3d74 100644 --- a/synapse/media/v0/content_repository.py +++ b/synapse/media/v0/content_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/__init__.py b/synapse/media/v1/__init__.py index e69de29bb2..619999d268 100644 --- a/synapse/media/v1/__init__.py +++ b/synapse/media/v1/__init__.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import PIL.Image + +# check for JPEG support. +try: + PIL.Image._getdecoder("rgb", "jpeg", None) +except IOError as e: + if str(e).startswith("decoder jpeg not available"): + raise Exception( + "FATAL: jpeg codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip install -I pillow'" + ) +except Exception: + # any other exception is fine + pass + + +# check for PNG support. +try: + PIL.Image._getdecoder("rgb", "zip", None) +except IOError as e: + if str(e).startswith("decoder zip not available"): + raise Exception( + "FATAL: zip codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip install -I pillow'" + ) +except Exception: + # any other exception is fine + pass diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 499be8cca0..688e7376ad 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -202,7 +202,8 @@ class BaseMediaResource(Resource): defer.returnValue(media_info) @defer.inlineCallbacks - def _respond_with_file(self, request, media_type, file_path): + def _respond_with_file(self, request, media_type, file_path, + file_size=None): logger.debug("Responding with %r", file_path) if os.path.isfile(file_path): @@ -216,13 +217,20 @@ class BaseMediaResource(Resource): request.setHeader( b"Cache-Control", b"public,max-age=86400,s-maxage=86400" ) + if file_size is None: + stat = os.stat(file_path) + file_size = stat.st_size + + request.setHeader( + b"Content-Length", b"%d" % (file_size,) + ) with open(file_path, "rb") as f: yield FileSender().beginFileTransfer(f, request) request.finish() else: - self._respond_404() + self._respond_404(request) def _get_thumbnail_requirements(self, media_type): if media_type == "image/jpeg": diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py index f3a6804e05..c585bb11f7 100644 --- a/synapse/media/v1/download_resource.py +++ b/synapse/media/v1/download_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -46,23 +46,29 @@ class DownloadResource(BaseMediaResource): def _respond_local_file(self, request, media_id): media_info = yield self.store.get_local_media(media_id) if not media_info: - self._respond_404() + self._respond_404(request) return media_type = media_info["media_type"] + media_length = media_info["media_length"] file_path = self.filepaths.local_media_filepath(media_id) - yield self._respond_with_file(request, media_type, file_path) + yield self._respond_with_file( + request, media_type, file_path, media_length + ) @defer.inlineCallbacks def _respond_remote_file(self, request, server_name, media_id): media_info = yield self._get_remote_media(server_name, media_id) media_type = media_info["media_type"] + media_length = media_info["media_length"] filesystem_id = media_info["filesystem_id"] file_path = self.filepaths.remote_media_filepath( server_name, filesystem_id ) - yield self._respond_with_file(request, media_type, file_path) + yield self._respond_with_file( + request, media_type, file_path, media_length + ) diff --git a/synapse/media/v1/filepath.py b/synapse/media/v1/filepath.py index 0078bc3d40..ed9a58e9d9 100644 --- a/synapse/media/v1/filepath.py +++ b/synapse/media/v1/filepath.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index 2070ec3c7f..461cc001f1 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py index 5ddcf54b9f..84f5e3463c 100644 --- a/synapse/media/v1/thumbnail_resource.py +++ b/synapse/media/v1/thumbnail_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -100,11 +100,12 @@ class ThumbnailResource(BaseMediaResource): t_type = thumbnail_info["thumbnail_type"] t_method = thumbnail_info["thumbnail_method"] file_id = thumbnail_info["filesystem_id"] + t_length = thumbnail_info["thumbnail_length"] file_path = self.filepaths.remote_media_thumbnail( server_name, file_id, t_width, t_height, t_type, t_method, ) - yield self._respond_with_file(request, t_type, file_path) + yield self._respond_with_file(request, t_type, file_path, t_length) else: yield self._respond_default_thumbnail( request, media_info, width, height, method, m_type, @@ -139,11 +140,12 @@ class ThumbnailResource(BaseMediaResource): t_height = thumbnail_info["thumbnail_height"] t_type = thumbnail_info["thumbnail_type"] t_method = thumbnail_info["thumbnail_method"] + t_length = thumbnail_info["thumbnail_length"] file_path = self.filepaths.default_thumbnail( top_level_type, sub_type, t_width, t_height, t_type, t_method, ) - yield self.respond_with_file(request, t_type, file_path) + yield self.respond_with_file(request, t_type, file_path, t_length) def _select_thumbnail(self, desired_width, desired_height, desired_method, desired_type, thumbnail_infos): diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py index 774ae4538f..bc86efea8f 100644 --- a/synapse/media/v1/thumbnailer.py +++ b/synapse/media/v1/thumbnailer.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ class Thumbnailer(object): def scale(self, output_path, width, height, output_type): """Rescales the image to the given dimensions""" - scaled = self.image.resize((width, height), Image.BILINEAR) + scaled = self.image.resize((width, height), Image.ANTIALIAS) return self.save_image(scaled, output_type, output_path) def crop(self, output_path, width, height, output_type): @@ -65,7 +65,7 @@ class Thumbnailer(object): if width * self.height > height * self.width: scaled_height = (width * self.height) // self.width scaled_image = self.image.resize( - (width, scaled_height), Image.BILINEAR + (width, scaled_height), Image.ANTIALIAS ) crop_top = (scaled_height - height) // 2 crop_bottom = height + crop_top @@ -73,7 +73,7 @@ class Thumbnailer(object): else: scaled_width = (height * self.width) // self.height scaled_image = self.image.resize( - (scaled_width, height), Image.BILINEAR + (scaled_width, height), Image.ANTIALIAS ) crop_left = (scaled_width - width) // 2 crop_right = width + crop_left diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index 5645b0df46..b1718a630b 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/notifier.py b/synapse/notifier.py index 383230caf1..b9d52d0c4c 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index a59630ec96..88ec9cd27d 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/admin.py b/synapse/rest/admin.py index d74c551512..0aa83514c8 100644 --- a/synapse/rest/admin.py +++ b/synapse/rest/admin.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/base.py b/synapse/rest/base.py index 06eda2587c..c583945527 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 868fa7abed..7ff44fdd9e 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/events.py b/synapse/rest/events.py index 3c1b041bfe..cf6d13f817 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/initial_sync.py b/synapse/rest/initial_sync.py index a1cb442256..a571589581 100644 --- a/synapse/rest/initial_sync.py +++ b/synapse/rest/initial_sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/login.py b/synapse/rest/login.py index 875da076af..6b8deff67b 100644 --- a/synapse/rest/login.py +++ b/synapse/rest/login.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index 9b42131628..ca4d2d21f0 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py index 351863ab5d..dc6eb424b0 100644 --- a/synapse/rest/profile.py +++ b/synapse/rest/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 4f0f5a7531..e3b26902d9 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 005a9f6f44..e40773758a 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/transactions.py b/synapse/rest/transactions.py index 31377bd41d..d933fea18a 100644 --- a/synapse/rest/transactions.py +++ b/synapse/rest/transactions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/voip.py b/synapse/rest/voip.py index 432c2475f8..011c35e69b 100644 --- a/synapse/rest/voip.py +++ b/synapse/rest/voip.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/server.py b/synapse/server.py index e4021481e8..c3bf46abbf 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/state.py b/synapse/state.py index 15397e2f38..8144fa02b4 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e6bb665932..4beb951b9f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -66,7 +66,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 10 +SCHEMA_VERSION = 11 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e0d97f440b..f660fc6eaf 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -64,7 +64,7 @@ class LoggingTransaction(object): # Don't let logging failures stop SQL from working pass - start = time.clock() * 1000 + start = time.time() * 1000 try: return self.txn.execute( sql, *args, **kwargs @@ -73,7 +73,7 @@ class LoggingTransaction(object): logger.exception("[SQL FAIL] {%s}", self.name) raise finally: - end = time.clock() * 1000 + end = time.time() * 1000 sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) @@ -93,7 +93,7 @@ class SQLBaseStore(object): def inner_func(txn, *args, **kwargs): with LoggingContext("runInteraction") as context: current_context.copy_to(context) - start = time.clock() * 1000 + start = time.time() * 1000 txn_id = SQLBaseStore._TXN_ID # We don't really need these to be unique, so lets stop it from @@ -109,7 +109,7 @@ class SQLBaseStore(object): logger.exception("[TXN FAIL] {%s}", name) raise finally: - end = time.clock() * 1000 + end = time.time() * 1000 transaction_logger.debug( "[TXN END] {%s} %f", name, end - start @@ -434,23 +434,31 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) - def _get_events(self, event_ids): + def _get_events(self, event_ids, check_redacted=True, + get_prev_content=False): return self.runInteraction( - "_get_events", self._get_events_txn, event_ids + "_get_events", self._get_events_txn, event_ids, + check_redacted=check_redacted, get_prev_content=get_prev_content, ) - def _get_events_txn(self, txn, event_ids): - events = [] - for e_id in event_ids: - ev = self._get_event_txn(txn, e_id) + def _get_events_txn(self, txn, event_ids, check_redacted=True, + get_prev_content=False): + if not event_ids: + return [] - if ev: - events.append(ev) + events = [ + self._get_event_txn( + txn, event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content + ) + for event_id in event_ids + ] - return events + return [e for e in events if e] def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=True): + get_prev_content=False): sql = ( "SELECT internal_metadata, json, r.event_id FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " @@ -467,6 +475,14 @@ class SQLBaseStore(object): internal_metadata, js, redacted = res + return self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + + def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, + check_redacted=True, get_prev_content=False): d = json.loads(js) internal_metadata = json.loads(internal_metadata) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 2be9c41374..68b7d59693 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index fb2eb21713..0cbcdd1b55 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py index 21511577c5..fcf011b234 100644 --- a/synapse/storage/feedback.py +++ b/synapse/storage/feedback.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index fd705138e6..1f244019fc 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index 18c068d3d9..7101d2beec 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 71b2bb084d..1dcd34723b 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 7e1fdd9d88..153c7ad027 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 4d15005c9e..75dffa4db2 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/room.py b/synapse/storage/room.py index c20abbfe4d..978b2c4a48 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 05b275663e..e59e65529b 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ from twisted.internet import defer +from collections import namedtuple + from ._base import SQLBaseStore from synapse.api.constants import Membership @@ -24,6 +26,12 @@ import logging logger = logging.getLogger(__name__) +RoomsForUser = namedtuple( + "RoomsForUser", + ("room_id", "sender", "membership") +) + + class RoomMemberStore(SQLBaseStore): def _store_room_member_txn(self, txn, event): @@ -123,6 +131,19 @@ class RoomMemberStore(SQLBaseStore): else: return None + def get_users_in_room(self, room_id): + def f(txn): + sql = ( + "SELECT m.user_id FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE m.membership = ? AND m.room_id = ?" + ) + + txn.execute(sql, (Membership.JOIN, room_id)) + return [r[0] for r in txn.fetchall()] + return self.runInteraction("get_users_in_room", f) + def get_room_members(self, room_id, membership=None): """Retrieve the current room member list for a room. @@ -150,19 +171,37 @@ class RoomMemberStore(SQLBaseStore): membership_list (list): A list of synapse.api.constants.Membership values which the user must be in. Returns: - A list of RoomMemberEvent objects + A list of dictionary objects, with room_id, membership and sender + defined. """ if not membership_list: return defer.succeed(None) - args = [user_id] - args.extend(membership_list) - where_clause = "user_id = ? AND (%s)" % ( " OR ".join(["membership = ?" for _ in membership_list]), ) - return self._get_members_query(where_clause, args) + args = [user_id] + args.extend(membership_list) + + def f(txn): + sql = ( + "SELECT m.room_id, m.sender, m.membership" + " FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE %s" + ) % (where_clause,) + + txn.execute(sql, args) + return [ + RoomsForUser(**r) for r in self.cursor_to_dict(txn) + ] + + return self.runInteraction( + "get_rooms_for_user_where_membership_is", + f + ) def get_joined_hosts_for_room(self, room_id): return self._simple_select_onecol( @@ -183,20 +222,14 @@ class RoomMemberStore(SQLBaseStore): ) def _get_members_query_txn(self, txn, where_clause, where_values): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "SELECT e.* FROM events as e " "INNER JOIN room_memberships as m " "ON e.event_id = m.event_id " "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " "WHERE %(where)s " ) % { - "redacted": del_sql, "where": where_clause, } @@ -206,26 +239,28 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results - @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a room. """ - user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) - sql = ( - "SELECT m.room_id FROM room_memberships as m " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE m.membership = 'join' " - "AND (%(clause)s) " - # TODO(paul): We've got duplicate rows in the database somewhere - # so we have to DISTINCT m.user_id here - "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" - ) % {"clause": user_list_clause} + def interaction(txn): + user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) + sql = ( + "SELECT m.room_id FROM room_memberships as m " + "INNER JOIN current_state_events as c " + "ON m.event_id = c.event_id " + "WHERE m.membership = 'join' " + "AND (%(clause)s) " + # TODO(paul): We've got duplicate rows in the database somewhere + # so we have to DISTINCT m.user_id here + "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" + ) % {"clause": user_list_clause} + + args = list(user_id_list) + args.append(len(user_id_list)) - args = list(user_id_list) - args.append(len(user_id_list)) + txn.execute(sql, args) - rows = yield self._execute(None, sql, *args) + return len(txn.fetchall()) > 0 - defer.returnValue(len(rows) > 0) + return self.runInteraction("user_rooms_intersect", interaction) diff --git a/synapse/storage/schema/delta/v11.sql b/synapse/storage/schema/delta/v11.sql new file mode 100644 index 0000000000..313592221b --- /dev/null +++ b/synapse/storage/schema/delta/v11.sql @@ -0,0 +1,16 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); \ No newline at end of file diff --git a/synapse/storage/schema/delta/v2.sql b/synapse/storage/schema/delta/v2.sql index 73b140465e..f740f6dd5d 100644 --- a/synapse/storage/schema/delta/v2.sql +++ b/synapse/storage/schema/delta/v2.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v3.sql b/synapse/storage/schema/delta/v3.sql index cade295989..c67e38ff52 100644 --- a/synapse/storage/schema/delta/v3.sql +++ b/synapse/storage/schema/delta/v3.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v4.sql b/synapse/storage/schema/delta/v4.sql index 25d2ead450..d3807b7686 100644 --- a/synapse/storage/schema/delta/v4.sql +++ b/synapse/storage/schema/delta/v4.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS redactions ( event_id TEXT NOT NULL, redacts TEXT NOT NULL, diff --git a/synapse/storage/schema/delta/v5.sql b/synapse/storage/schema/delta/v5.sql index af9df11aa9..0874a15431 100644 --- a/synapse/storage/schema/delta/v5.sql +++ b/synapse/storage/schema/delta/v5.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS user_ips ( user TEXT NOT NULL, diff --git a/synapse/storage/schema/delta/v6.sql b/synapse/storage/schema/delta/v6.sql index 9bf2068d84..a9e0a4fe0d 100644 --- a/synapse/storage/schema/delta/v6.sql +++ b/synapse/storage/schema/delta/v6.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v8.sql b/synapse/storage/schema/delta/v8.sql index daf6646ed5..1e9f8b18cb 100644 --- a/synapse/storage/schema/delta/v8.sql +++ b/synapse/storage/schema/delta/v8.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql index 0af29733a0..455d51a70c 100644 --- a/synapse/storage/schema/delta/v9.sql +++ b/synapse/storage/schema/delta/v9.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index be1c72a775..1e766d6db2 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id TEXT NOT NULL, diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/event_signatures.sql index b6b56b47a2..c28c39c48a 100644 --- a/synapse/storage/schema/event_signatures.sql +++ b/synapse/storage/schema/event_signatures.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 253f9f779b..dd00c1cd2f 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/keys.sql index 9bf2068d84..a9e0a4fe0d 100644 --- a/synapse/storage/schema/keys.sql +++ b/synapse/storage/schema/keys.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/media_repository.sql b/synapse/storage/schema/media_repository.sql index b785fa0208..afdf48cbfb 100644 --- a/synapse/storage/schema/media_repository.sql +++ b/synapse/storage/schema/media_repository.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/presence.sql b/synapse/storage/schema/presence.sql index 595b3b5a69..f9f8db9697 100644 --- a/synapse/storage/schema/presence.sql +++ b/synapse/storage/schema/presence.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/profiles.sql b/synapse/storage/schema/profiles.sql index 58209f1af0..f06a528b4d 100644 --- a/synapse/storage/schema/profiles.sql +++ b/synapse/storage/schema/profiles.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/redactions.sql b/synapse/storage/schema/redactions.sql index 4c2829d05d..5011d95db8 100644 --- a/synapse/storage/schema/redactions.sql +++ b/synapse/storage/schema/redactions.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS redactions ( event_id TEXT NOT NULL, redacts TEXT NOT NULL, diff --git a/synapse/storage/schema/room_aliases.sql b/synapse/storage/schema/room_aliases.sql index 9191016814..0d2df01603 100644 --- a/synapse/storage/schema/room_aliases.sql +++ b/synapse/storage/schema/room_aliases.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql index 2c48d6daca..1fe8f1e430 100644 --- a/synapse/storage/schema/state.sql +++ b/synapse/storage/schema/state.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index de461bfa15..2d30f99b06 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,7 @@ CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destinatio CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( destination ); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); diff --git a/synapse/storage/schema/users.sql b/synapse/storage/schema/users.sql index 8244f733bd..08ccfdac0a 100644 --- a/synapse/storage/schema/users.sql +++ b/synapse/storage/schema/users.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 3a705119fd..d0d53770f2 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/state.py b/synapse/storage/state.py index afe3e5edea..5327517704 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ from ._base import SQLBaseStore +import logging + +logger = logging.getLogger(__name__) + class StateStore(SQLBaseStore): """ Keeps track of the state at a given event. @@ -62,14 +66,8 @@ class StateStore(SQLBaseStore): keyvalues={"state_group": group}, retcol="event_id", ) - state = [] - for state_id in state_ids: - s = self._get_events_txn( - txn, - [state_id], - ) - if s: - state.extend(s) + + state = self._get_events_txn(txn, state_ids) res[group] = state diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3405cb365e..bedc3c6c52 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -137,7 +137,6 @@ class StreamStore(SQLBaseStore): with_feedback=with_feedback, ) - @defer.inlineCallbacks @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): @@ -157,11 +156,6 @@ class StreamStore(SQLBaseStore): "WHERE m.user_id = ? " ) - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - if limit: limit = max(limit, MAX_STREAM_SIZE) else: @@ -172,38 +166,42 @@ class StreamStore(SQLBaseStore): to_id = _parse_stream_token(to_key) if from_key == to_key: - defer.returnValue(([], to_key)) - return + return defer.succeed(([], to_key)) sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events AS e WHERE " + "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { - "redacted": del_sql, "current": current_room_membership_sql, "invites": membership_sql, "limit": limit } - rows = yield self._execute_and_decode( - sql, - user_id, user_id, from_id, to_id - ) + def f(txn): + txn.execute(sql, (user_id, user_id, from_id, to_id,)) - ret = yield self._parse_events(rows) + rows = self.cursor_to_dict(txn) - if rows: - key = "s%d" % max([r["stream_ordering"] for r in rows]) - else: - # Assume we didn't get anything because there was nothing to get. - key = to_key + ret = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) + + if rows: + key = "s%d" % max([r["stream_ordering"] for r in rows]) + else: + # Assume we didn't get anything because there was nothing to + # get. + key = to_key + + return ret, key - defer.returnValue((ret, key)) + return self.runInteraction("get_room_events_stream", f) - @defer.inlineCallbacks @log_function def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, @@ -221,7 +219,9 @@ class StreamStore(SQLBaseStore): bounds = _get_token_bound(from_key, from_comp) if to_key: - bounds = "%s AND %s" % (bounds, _get_token_bound(to_key, to_comp)) + bounds = "%s AND %s" % ( + bounds, _get_token_bound(to_key, to_comp) + ) if int(limit) > 0: args.append(int(limit)) @@ -229,87 +229,78 @@ class StreamStore(SQLBaseStore): else: limit_str = "" - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = events.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events" + "SELECT * FROM events" " WHERE outlier = 0 AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" ) % { - "redacted": del_sql, "bounds": bounds, "order": order, "limit": limit_str } - rows = yield self._execute_and_decode( - sql, - *args - ) - - if rows: - topo = rows[-1]["topological_ordering"] - toke = rows[-1]["stream_ordering"] - if direction == 'b': - topo -= 1 - toke -= 1 - next_token = "t%s-%s" % (topo, toke) - else: - # TODO (erikj): We should work out what to do here instead. - next_token = to_key if to_key else from_key + def f(txn): + txn.execute(sql, args) + + rows = self.cursor_to_dict(txn) + + if rows: + topo = rows[-1]["topological_ordering"] + toke = rows[-1]["stream_ordering"] + if direction == 'b': + topo -= 1 + toke -= 1 + next_token = "t%s-%s" % (topo, toke) + else: + # TODO (erikj): We should work out what to do here instead. + next_token = to_key if to_key else from_key + + events = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) - events = yield self._parse_events(rows) + return events, next_token, - defer.returnValue( - ( - events, - next_token - ) - ) + return self.runInteraction("paginate_room_events", f) - @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, end_token, with_feedback=False): # TODO (erikj): Handle compressed feedback - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = events.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events " + "SELECT stream_ordering, topological_ordering, event_id FROM events " "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) % { - "redacted": del_sql, - } - - rows = yield self._execute_and_decode( - sql, - room_id, end_token, limit ) - rows.reverse() # As we selected with reverse ordering + def f(txn): + txn.execute(sql, (room_id, end_token, limit,)) - if rows: - topo = rows[0]["topological_ordering"] - toke = rows[0]["stream_ordering"] - start_token = "t%s-%s" % (topo, toke) + rows = self.cursor_to_dict(txn) - token = (start_token, end_token) - else: - token = (end_token, end_token) + rows.reverse() # As we selected with reverse ordering - events = yield self._parse_events(rows) + if rows: + topo = rows[0]["topological_ordering"] + toke = rows[0]["stream_ordering"] + start_token = "t%s-%s" % (topo, toke) + + token = (start_token, end_token) + else: + token = (end_token, end_token) + + events = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) - ret = (events, token) + return events, token - defer.returnValue(ret) + return self.runInteraction("get_recent_events_for_room", f) def get_room_events_max_id(self): return self.runInteraction( diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 423cc3f02a..e06ef35690 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/__init__.py b/synapse/streams/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/streams/__init__.py +++ b/synapse/streams/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/config.py b/synapse/streams/config.py index 2114c940e7..2ec7c5403b 100644 --- a/synapse/streams/config.py +++ b/synapse/streams/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/events.py b/synapse/streams/events.py index fb698d2d71..5c8e54b78b 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/types.py b/synapse/types.py index 7c533193e1..faac729ff2 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 7ec5033ceb..4e837a918e 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/async.py b/synapse/util/async.py index 7dd3ec3a72..c4fe5d522f 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 6925ac96b6..9d9c350397 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/emailutils.py b/synapse/util/emailutils.py index 7038cab6c2..7f9a77bf44 100644 --- a/synapse/util/emailutils.py +++ b/synapse/util/emailutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index fcfb38b3b4..a13a2015e4 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,10 +17,11 @@ from frozendict import frozendict def freeze(o): - if isinstance(o, dict) or isinstance(o, frozendict): + t = type(o) + if t is dict: return frozendict({k: freeze(v) for k, v in o.items()}) - if isinstance(o, basestring): + if t is str or t is unicode: return o try: diff --git a/synapse/util/jsonobject.py b/synapse/util/jsonobject.py index e79b68f661..0765f7d217 100644 --- a/synapse/util/jsonobject.py +++ b/synapse/util/jsonobject.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/lockutils.py b/synapse/util/lockutils.py index 3a84c09db4..33edc5c20e 100644 --- a/synapse/util/lockutils.py +++ b/synapse/util/lockutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 7d85018d97..da7872e95d 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -1,3 +1,17 @@ +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import threading import logging diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index 903a6cf1b3..fd9ac4d4d4 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,14 +19,37 @@ from functools import wraps import logging import inspect +import time + + +_TIME_FUNC_ID = 0 + + +def _log_debug_as_f(f, msg, msg_args): + name = f.__module__ + logger = logging.getLogger(name) + + if logger.isEnabledFor(logging.DEBUG): + lineno = f.func_code.co_firstlineno + pathname = f.func_code.co_filename + + record = logging.LogRecord( + name=name, + level=logging.DEBUG, + pathname=pathname, + lineno=lineno, + msg=msg, + args=msg_args, + exc_info=None + ) + + logger.handle(record) def log_function(f): """ Function decorator that logs every call to that function. """ func_name = f.__name__ - lineno = f.func_code.co_firstlineno - pathname = f.func_code.co_filename @wraps(f) def wrapped(*args, **kwargs): @@ -52,24 +75,50 @@ def log_function(f): "args": ", ".join(func_args) } - record = logging.LogRecord( - name=name, - level=level, - pathname=pathname, - lineno=lineno, - msg="Invoked '%(func_name)s' with args: %(args)s", - args=msg_args, - exc_info=None + _log_debug_as_f( + f, + "Invoked '%(func_name)s' with args: %(args)s", + msg_args ) - logger.handle(record) - return f(*args, **kwargs) wrapped.__name__ = func_name return wrapped +def time_function(f): + func_name = f.__name__ + + @wraps(f) + def wrapped(*args, **kwargs): + global _TIME_FUNC_ID + id = _TIME_FUNC_ID + _TIME_FUNC_ID += 1 + + start = time.clock() * 1000 + + try: + _log_debug_as_f( + f, + "[FUNC START] {%s-%d}", + (func_name, id), + ) + + r = f(*args, **kwargs) + finally: + end = time.clock() * 1000 + _log_debug_as_f( + f, + "[FUNC END] {%s-%d} %f", + (func_name, id, end-start,), + ) + + return r + + return wrapped + + def trace_function(f): func_name = f.__name__ linenum = f.func_code.co_firstlineno diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index 8767e437dd..ea53a8085c 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. |