From 613748804a302ced5de124c86606aff3a36acf42 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Wed, 9 Dec 2015 17:35:55 +0000
Subject: Changelog for v0.12.0
---
CHANGES.rst | 34 ++++++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/CHANGES.rst b/CHANGES.rst
index 5c38c1915f..d151badc98 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,37 @@
+Changes in synapse v0.12.0 (2015-12-09)
+=======================================
+
+* Host the client APIs released as r0 by
+ https://matrix.org/docs/spec/r0.0.0/client_server.html
+ on paths prefixed by /_matrix/client/r0. (PR #430, PR #415, PR #400)
+* Updates the client APIs to match r0 of the matrix specification.
+
+ * All APIs return events in the new event format, old APIs also include
+ the fields needed to parse the event using the old format for
+ compatibility. (PR #402)
+ * Search results are now given as a JSON array rather than
+ a JSON object (PR #405)
+ * Miscellaneous changes to search (PR #403, PR #406, PR #412)
+ * Filter JSON objects may now be passed as query parameters to /sync
+ (PR #431)
+ * Fix implementation of /admin/whois (PR #418)
+ * Only include the rooms that user has left in /sync if the client requests
+ them in the filter (PR #423)
+ * Don't push for m.room.message by default (PR #411)
+ * Add API for setting per account user data (PR #392)
+ * Allow users to forget rooms (PR #385)
+
+* Performance improvements and monitoring:
+
+ * Add per-request counters for CPU time spent on the main python thread.
+ (PR #421, PR #420)
+ * Add per-request counters for time spent in the database (PR #429)
+ * Make state updates in the C+S API idempotent (PR #416)
+ * Only fire user_joined_room if the user has actually joined. (PR #410)
+ * Reuse a single http client, rather than creating new ones (PR #413)
+
+* Fixed a bug upgrading from older versions of synapse on postgresql (PR #417)
+
Changes in synapse v0.11.1 (2015-11-20)
=======================================
--
cgit 1.5.1
From 5bdb93c2a6b4efcd25bb9a5974f9c4eebb040b23 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Wed, 9 Dec 2015 17:45:35 +0000
Subject: Add to changelog
---
CHANGES.rst | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index d151badc98..03dc975762 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -3,7 +3,7 @@ Changes in synapse v0.12.0 (2015-12-09)
* Host the client APIs released as r0 by
https://matrix.org/docs/spec/r0.0.0/client_server.html
- on paths prefixed by /_matrix/client/r0. (PR #430, PR #415, PR #400)
+ on paths prefixed by ``/_matrix/client/r0``. (PR #430, PR #415, PR #400)
* Updates the client APIs to match r0 of the matrix specification.
* All APIs return events in the new event format, old APIs also include
@@ -12,12 +12,12 @@ Changes in synapse v0.12.0 (2015-12-09)
* Search results are now given as a JSON array rather than
a JSON object (PR #405)
* Miscellaneous changes to search (PR #403, PR #406, PR #412)
- * Filter JSON objects may now be passed as query parameters to /sync
+ * Filter JSON objects may now be passed as query parameters to ``/sync``
(PR #431)
- * Fix implementation of /admin/whois (PR #418)
- * Only include the rooms that user has left in /sync if the client requests
- them in the filter (PR #423)
- * Don't push for m.room.message by default (PR #411)
+ * Fix implementation of ``/admin/whois`` (PR #418)
+ * Only include the rooms that user has left in ``/sync`` if the client
+ requests them in the filter (PR #423)
+ * Don't push for ``m.room.message`` by default (PR #411)
* Add API for setting per account user data (PR #392)
* Allow users to forget rooms (PR #385)
@@ -27,7 +27,7 @@ Changes in synapse v0.12.0 (2015-12-09)
(PR #421, PR #420)
* Add per-request counters for time spent in the database (PR #429)
* Make state updates in the C+S API idempotent (PR #416)
- * Only fire user_joined_room if the user has actually joined. (PR #410)
+ * Only fire ``user_joined_room`` if the user has actually joined. (PR #410)
* Reuse a single http client, rather than creating new ones (PR #413)
* Fixed a bug upgrading from older versions of synapse on postgresql (PR #417)
--
cgit 1.5.1
From 05f6cb42db1cc1a9720fa7214a14a26613a8b699 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Wed, 9 Dec 2015 17:48:02 +0000
Subject: Bump synapse version to v0.12.0
---
synapse/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 3e7e26bf60..5db4eae354 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
""" This is a reference implementation of a Matrix home server.
"""
-__version__ = "0.11.1"
+__version__ = "0.12.0"
--
cgit 1.5.1
From dd9430e758ed103af8883392e0bc4cc0ac600a4c Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Thu, 10 Dec 2015 11:26:58 +0000
Subject: Update release date
---
CHANGES.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index 03dc975762..6247d1b389 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,4 +1,4 @@
-Changes in synapse v0.12.0 (2015-12-09)
+Changes in synapse v0.12.0 (2015-12-10)
=======================================
* Host the client APIs released as r0 by
--
cgit 1.5.1
From a8589d1ff3ca9f473ffa492e0c96778333928882 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Thu, 10 Dec 2015 11:39:00 +0000
Subject: Mark the version as a -rc1 release candidate
---
CHANGES.rst | 4 ++--
synapse/__init__.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index 6247d1b389..f81a51dc7f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,5 +1,5 @@
-Changes in synapse v0.12.0 (2015-12-10)
-=======================================
+Changes in synapse v0.12.0-rc1 (2015-12-10)
+===========================================
* Host the client APIs released as r0 by
https://matrix.org/docs/spec/r0.0.0/client_server.html
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 5db4eae354..c357f8f9c2 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
""" This is a reference implementation of a Matrix home server.
"""
-__version__ = "0.12.0"
+__version__ = "0.12.0-rc1"
--
cgit 1.5.1
From 99afb4b750f9ba5074f8e7dd79144cf678c668f1 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 10 Dec 2015 17:08:21 +0000
Subject: Ensure that the event that gets persisted is the one that was signed
---
synapse/handlers/federation.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 2855f2d7c3..e7ad48c948 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -596,7 +596,7 @@ class FederationHandler(BaseHandler):
handled_events = set()
try:
- new_event = self._sign_event(event)
+ event = self._sign_event(event)
# Try the host we successfully got a response to /make_join/
# request first.
try:
@@ -604,7 +604,7 @@ class FederationHandler(BaseHandler):
target_hosts.insert(0, origin)
except ValueError:
pass
- ret = yield self.replication_layer.send_join(target_hosts, new_event)
+ ret = yield self.replication_layer.send_join(target_hosts, event)
origin = ret["origin"]
state = ret["state"]
@@ -613,12 +613,12 @@ class FederationHandler(BaseHandler):
handled_events.update([s.event_id for s in state])
handled_events.update([a.event_id for a in auth_chain])
- handled_events.add(new_event.event_id)
+ handled_events.add(event.event_id)
logger.debug("do_invite_join auth_chain: %s", auth_chain)
logger.debug("do_invite_join state: %s", state)
- logger.debug("do_invite_join event: %s", new_event)
+ logger.debug("do_invite_join event: %s", event)
try:
yield self.store.store_room(
@@ -636,14 +636,14 @@ class FederationHandler(BaseHandler):
with PreserveLoggingContext():
d = self.notifier.on_new_room_event(
- new_event, event_stream_id, max_stream_id,
+ event, event_stream_id, max_stream_id,
extra_users=[joinee]
)
def log_failure(f):
logger.warn(
"Failed to notify about %s: %s",
- new_event.event_id, f.value
+ event.event_id, f.value
)
d.addErrback(log_failure)
--
cgit 1.5.1
From 7d6b3133125aef802dad36d120ad23d5e33948bf Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 10 Dec 2015 17:49:34 +0000
Subject: Add caches for whether a room has been forgotten by a user
---
synapse/handlers/room.py | 2 +-
synapse/storage/roommember.py | 11 +++++++----
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 116a998c42..a72c3fda9f 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -755,7 +755,7 @@ class RoomMemberHandler(BaseHandler):
defer.returnValue((token, public_key, key_validity_url, display_name))
def forget(self, user, room_id):
- self.store.forget(user.to_string(), room_id)
+ return self.store.forget(user.to_string(), room_id)
class RoomListHandler(BaseHandler):
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 69398b7c8e..e1777d7afa 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
from collections import namedtuple
from ._base import SQLBaseStore
-from synapse.util.caches.descriptors import cached
+from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
from synapse.api.constants import Membership
from synapse.types import UserID
@@ -270,6 +270,7 @@ class RoomMemberStore(SQLBaseStore):
defer.returnValue(ret)
+ @defer.inlineCallbacks
def forget(self, user_id, room_id):
"""Indicate that user_id wishes to discard history for room_id."""
def f(txn):
@@ -284,9 +285,11 @@ class RoomMemberStore(SQLBaseStore):
" room_id = ?"
)
txn.execute(sql, (user_id, room_id))
- self.runInteraction("forget_membership", f)
+ yield self.runInteraction("forget_membership", f)
+ self.was_forgotten_at.invalidate_all()
+ self.did_forget.invalidate((user_id, room_id))
- @defer.inlineCallbacks
+ @cachedInlineCallbacks(num_args=2)
def did_forget(self, user_id, room_id):
"""Returns whether user_id has elected to discard history for room_id.
@@ -310,7 +313,7 @@ class RoomMemberStore(SQLBaseStore):
count = yield self.runInteraction("did_forget_membership", f)
defer.returnValue(count == 0)
- @defer.inlineCallbacks
+ @cachedInlineCallbacks(num_args=3)
def was_forgotten_at(self, user_id, room_id, event_id):
"""Returns whether user_id has elected to discard history for room_id at event_id.
--
cgit 1.5.1
From 515548a47ae0418203224a4315b88531cf28a9de Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 10 Dec 2015 17:54:23 +0000
Subject: Missing yield
---
synapse/rest/client/v1/room.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 53cc29becb..6fe53f70e5 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -490,7 +490,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet):
)
if membership_action == "forget":
- self.handlers.room_member_handler.forget(user, room_id)
+ yield self.handlers.room_member_handler.forget(user, room_id)
defer.returnValue((200, {}))
--
cgit 1.5.1
From 5577a6109052e6c953a1532ecb3b473db709905e Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 10 Dec 2015 19:03:06 +0000
Subject: throwaway 1-liner for generating password hashes
---
scripts/gen_password | 1 +
1 file changed, 1 insertion(+)
create mode 100644 scripts/gen_password
diff --git a/scripts/gen_password b/scripts/gen_password
new file mode 100644
index 0000000000..7afd3a5dfd
--- /dev/null
+++ b/scripts/gen_password
@@ -0,0 +1 @@
+perl -MCrypt::Random -MCrypt::Eksblowfish::Bcrypt -e 'print Crypt::Eksblowfish::Bcrypt::bcrypt("secret", "\$2\$12\$" . Crypt::Eksblowfish::Bcrypt::en_base64(Crypt::Random::makerandom_octet(Length=>16)))."\n"'
--
cgit 1.5.1
From 51fb590c0e787c385bea1d595fa8bceea23c26e5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 11 Dec 2015 11:12:57 +0000
Subject: Use more efficient query form
---
synapse/storage/search.py | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 39f600f53c..c39d54a7ca 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -143,7 +143,7 @@ class SearchStore(BackgroundUpdateStore):
search_query = search_query = _parse_query(self.database_engine, search_term)
- args = [search_query]
+ args = []
# Make sure we don't explode because the person is in too many rooms.
# We filter the results below regardless.
@@ -164,16 +164,19 @@ class SearchStore(BackgroundUpdateStore):
if isinstance(self.database_engine, PostgresEngine):
sql = (
- "SELECT ts_rank_cd(vector, query) AS rank, room_id, event_id"
- " FROM to_tsquery('english', ?) as query, event_search"
- " WHERE vector @@ query"
+ "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank,"
+ " room_id, event_id"
+ " FROM event_search"
+ " WHERE vector @@ to_tsquery('english', ?)"
)
+ args = [search_query, search_query] + args
elif isinstance(self.database_engine, Sqlite3Engine):
sql = (
"SELECT rank(matchinfo(event_search)) as rank, room_id, event_id"
" FROM event_search"
" WHERE value MATCH ?"
)
+ args = [search_query] + args
else:
# This should be unreachable.
raise Exception("Unrecognized database engine")
@@ -232,7 +235,7 @@ class SearchStore(BackgroundUpdateStore):
search_query = search_query = _parse_query(self.database_engine, search_term)
- args = [search_query]
+ args = []
# Make sure we don't explode because the person is in too many rooms.
# We filter the results below regardless.
@@ -267,12 +270,13 @@ class SearchStore(BackgroundUpdateStore):
if isinstance(self.database_engine, PostgresEngine):
sql = (
- "SELECT ts_rank_cd(vector, query) as rank,"
+ "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank,"
" origin_server_ts, stream_ordering, room_id, event_id"
- " FROM to_tsquery('english', ?) as query, event_search"
+ " FROM event_search"
" NATURAL JOIN events"
- " WHERE vector @@ query AND "
+ " WHERE vector @@ to_tsquery('english', ?) AND "
)
+ args = [search_term, search_term] + args
elif isinstance(self.database_engine, Sqlite3Engine):
# We use CROSS JOIN here to ensure we use the right indexes.
# https://sqlite.org/optoverview.html#crossjoin
@@ -292,6 +296,7 @@ class SearchStore(BackgroundUpdateStore):
" CROSS JOIN events USING (event_id)"
" WHERE "
)
+ args = [search_term] + args
else:
# This should be unreachable.
raise Exception("Unrecognized database engine")
--
cgit 1.5.1
From d9a5c56930c22b02268f5deca4df84eba345ec2c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 11 Dec 2015 11:40:23 +0000
Subject: Include approximate count of search results
---
synapse/handlers/search.py | 8 ++++++-
synapse/storage/search.py | 56 ++++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index bc79564287..99ef56871c 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -152,11 +152,15 @@ class SearchHandler(BaseHandler):
highlights = set()
+ count = None
+
if order_by == "rank":
search_result = yield self.store.search_msgs(
room_ids, search_term, keys
)
+ count = search_result["count"]
+
if search_result["highlights"]:
highlights.update(search_result["highlights"])
@@ -207,6 +211,8 @@ class SearchHandler(BaseHandler):
if search_result["highlights"]:
highlights.update(search_result["highlights"])
+ count = search_result["count"]
+
results = search_result["results"]
results_map = {r["event"].event_id: r for r in results}
@@ -359,7 +365,7 @@ class SearchHandler(BaseHandler):
rooms_cat_res = {
"results": results,
- "count": len(results),
+ "count": count,
"highlights": list(highlights),
}
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index c39d54a7ca..efd87d99bb 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -162,6 +162,9 @@ class SearchStore(BackgroundUpdateStore):
"(%s)" % (" OR ".join(local_clauses),)
)
+ count_args = args
+ count_clauses = clauses
+
if isinstance(self.database_engine, PostgresEngine):
sql = (
"SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank,"
@@ -170,6 +173,12 @@ class SearchStore(BackgroundUpdateStore):
" WHERE vector @@ to_tsquery('english', ?)"
)
args = [search_query, search_query] + args
+
+ count_sql = (
+ "SELECT room_id, count(*) as count FROM event_search"
+ " WHERE vector @@ to_tsquery('english', ?)"
+ )
+ count_args = [search_query] + count_args
elif isinstance(self.database_engine, Sqlite3Engine):
sql = (
"SELECT rank(matchinfo(event_search)) as rank, room_id, event_id"
@@ -177,6 +186,12 @@ class SearchStore(BackgroundUpdateStore):
" WHERE value MATCH ?"
)
args = [search_query] + args
+
+ count_sql = (
+ "SELECT room_id, count(*) as count FROM event_search"
+ " WHERE value MATCH ? AND "
+ )
+ count_args = [search_term] + count_args
else:
# This should be unreachable.
raise Exception("Unrecognized database engine")
@@ -184,6 +199,9 @@ class SearchStore(BackgroundUpdateStore):
for clause in clauses:
sql += " AND " + clause
+ for clause in count_clauses:
+ count_sql += " AND " + clause
+
# We add an arbitrary limit here to ensure we don't try to pull the
# entire table from the database.
sql += " ORDER BY rank DESC LIMIT 500"
@@ -205,6 +223,14 @@ class SearchStore(BackgroundUpdateStore):
if isinstance(self.database_engine, PostgresEngine):
highlights = yield self._find_highlights_in_postgres(search_query, events)
+ count_sql += " GROUP BY room_id"
+
+ count_results = yield self._execute(
+ "search_rooms_count", self.cursor_to_dict, count_sql, *count_args
+ )
+
+ count = sum(row["count"] for row in count_results if row["room_id"] in room_ids)
+
defer.returnValue({
"results": [
{
@@ -215,6 +241,7 @@ class SearchStore(BackgroundUpdateStore):
if r["event_id"] in event_map
],
"highlights": highlights,
+ "count": count,
})
@defer.inlineCallbacks
@@ -254,6 +281,9 @@ class SearchStore(BackgroundUpdateStore):
"(%s)" % (" OR ".join(local_clauses),)
)
+ count_args = args
+ count_clauses = clauses
+
if pagination_token:
try:
origin_server_ts, stream = pagination_token.split(",")
@@ -276,7 +306,13 @@ class SearchStore(BackgroundUpdateStore):
" NATURAL JOIN events"
" WHERE vector @@ to_tsquery('english', ?) AND "
)
- args = [search_term, search_term] + args
+ args = [search_query, search_query] + args
+
+ count_sql = (
+ "SELECT room_id, count(*) as count FROM event_search"
+ " WHERE vector @@ to_tsquery('english', ?) AND "
+ )
+ count_args = [search_query] + count_args
elif isinstance(self.database_engine, Sqlite3Engine):
# We use CROSS JOIN here to ensure we use the right indexes.
# https://sqlite.org/optoverview.html#crossjoin
@@ -296,12 +332,19 @@ class SearchStore(BackgroundUpdateStore):
" CROSS JOIN events USING (event_id)"
" WHERE "
)
- args = [search_term] + args
+ args = [search_query] + args
+
+ count_sql = (
+ "SELECT room_id, count(*) as count FROM event_search"
+ " WHERE value MATCH ? AND "
+ )
+ count_args = [search_term] + count_args
else:
# This should be unreachable.
raise Exception("Unrecognized database engine")
sql += " AND ".join(clauses)
+ count_sql += " AND ".join(count_clauses)
# We add an arbitrary limit here to ensure we don't try to pull the
# entire table from the database.
@@ -326,6 +369,14 @@ class SearchStore(BackgroundUpdateStore):
if isinstance(self.database_engine, PostgresEngine):
highlights = yield self._find_highlights_in_postgres(search_query, events)
+ count_sql += " GROUP BY room_id"
+
+ count_results = yield self._execute(
+ "search_rooms_count", self.cursor_to_dict, count_sql, *count_args
+ )
+
+ count = sum(row["count"] for row in count_results if row["room_id"] in room_ids)
+
defer.returnValue({
"results": [
{
@@ -339,6 +390,7 @@ class SearchStore(BackgroundUpdateStore):
if r["event_id"] in event_map
],
"highlights": highlights,
+ "count": count,
})
def _find_highlights_in_postgres(self, search_query, events):
--
cgit 1.5.1
From 5a3e4e43d893d73f3a6b3eab985a6482c8e33e78 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 11 Dec 2015 11:38:03 +0000
Subject: SYN-90: We don't need --proccess-dependency-links
When installing synapse since all its dependencies are on PyPI
---
README.rst | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/README.rst b/README.rst
index 80e1b26e60..06f06fd353 100644
--- a/README.rst
+++ b/README.rst
@@ -130,7 +130,7 @@ To install the synapse homeserver run::
virtualenv -p python2.7 ~/.synapse
source ~/.synapse/bin/activate
pip install --upgrade setuptools
- pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master
+ pip install https://github.com/matrix-org/synapse/tarball/master
This installs synapse, along with the libraries it uses, into a virtual
environment under ``~/.synapse``. Feel free to pick a different directory
@@ -235,8 +235,7 @@ pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 )::
You also may need to explicitly specify python 2.7 again during the install
request::
- pip2.7 install --process-dependency-links \
- https://github.com/matrix-org/synapse/tarball/master
+ pip2.7 install https://github.com/matrix-org/synapse/tarball/master
If you encounter an error with lib bcrypt causing an Wrong ELF Class:
ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
@@ -295,8 +294,7 @@ Troubleshooting
Troubleshooting Installation
----------------------------
-Synapse requires pip 1.7 or later, so if your OS provides too old a version and
-you get errors about ``error: no such option: --process-dependency-links`` you
+Synapse requires pip 1.7 or later, so if your OS provides too old a version you
may need to manually upgrade it::
sudo pip install --upgrade pip
--
cgit 1.5.1
From 1ee7280c4c7a6ad99236a10a861fde3cd013892b Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 11 Dec 2015 16:48:20 +0000
Subject: Do the /sync in parallel accross the rooms like /initialSync does
---
synapse/handlers/sync.py | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 24c2b2fad6..7088c20cb4 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -17,6 +17,7 @@ from ._base import BaseHandler
from synapse.streams.config import PaginationConfig
from synapse.api.constants import Membership, EventTypes
+from synapse.util import unwrapFirstError
from twisted.internet import defer
@@ -209,9 +210,10 @@ class SyncHandler(BaseHandler):
joined = []
invited = []
archived = []
+ deferreds = []
for event in room_list:
if event.membership == Membership.JOIN:
- room_sync = yield self.full_state_sync_for_joined_room(
+ room_sync_deferred = self.full_state_sync_for_joined_room(
room_id=event.room_id,
sync_config=sync_config,
now_token=now_token,
@@ -220,7 +222,8 @@ class SyncHandler(BaseHandler):
tags_by_room=tags_by_room,
account_data_by_room=account_data_by_room,
)
- joined.append(room_sync)
+ room_sync_deferred.addCallback(joined.append)
+ deferreds.append(room_sync_deferred)
elif event.membership == Membership.INVITE:
invite = yield self.store.get_event(event.event_id)
invited.append(InvitedSyncResult(
@@ -231,7 +234,7 @@ class SyncHandler(BaseHandler):
leave_token = now_token.copy_and_replace(
"room_key", "s%d" % (event.stream_ordering,)
)
- room_sync = yield self.full_state_sync_for_archived_room(
+ room_sync_deferred = self.full_state_sync_for_archived_room(
sync_config=sync_config,
room_id=event.room_id,
leave_event_id=event.event_id,
@@ -240,7 +243,12 @@ class SyncHandler(BaseHandler):
tags_by_room=tags_by_room,
account_data_by_room=account_data_by_room,
)
- archived.append(room_sync)
+ room_sync_deferred.addCallback(archived.append)
+ deferreds.append(room_sync_deferred)
+
+ yield defer.gatherResults(
+ deferreds, consumeErrors=True
+ ).addErrback(unwrapFirstError)
defer.returnValue(SyncResult(
presence=presence,
--
cgit 1.5.1
From bfc52a2342999a7887dcc5ba653b67454c0fc2c8 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 14 Dec 2015 11:38:11 +0000
Subject: Fix typo in sql for full text search on sqlite3
---
synapse/storage/search.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index efd87d99bb..00f89ff02f 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -189,7 +189,7 @@ class SearchStore(BackgroundUpdateStore):
count_sql = (
"SELECT room_id, count(*) as count FROM event_search"
- " WHERE value MATCH ? AND "
+ " WHERE value MATCH ?"
)
count_args = [search_term] + count_args
else:
--
cgit 1.5.1
From 76e69cc8de186c42be5763be0492d074319060cc Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dawagner@gmail.com>
Date: Mon, 14 Dec 2015 12:38:55 +0000
Subject: Fix typo
---
synapse/storage/roommember.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index e1777d7afa..4e0e9ab59a 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -121,7 +121,7 @@ class RoomMemberStore(SQLBaseStore):
return self.get_rooms_for_user_where_membership_is(
user_id, [Membership.INVITE]
).addCallback(lambda invites: self._get_events([
- invites.event_id for invite in invites
+ invite.event_id for invite in invites
]))
def get_leave_and_ban_events_for_user(self, user_id):
--
cgit 1.5.1
From 338c0a8a69096c188f4739c235f74a072a62e92f Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dawagner@gmail.com>
Date: Mon, 14 Dec 2015 13:50:50 +0000
Subject: Include errcode on Internal Server Error
---
synapse/http/server.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/synapse/http/server.py b/synapse/http/server.py
index c44bdfc888..1b936b6892 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -15,7 +15,7 @@
from synapse.api.errors import (
- cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError
+ cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes
)
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
import synapse.metrics
@@ -127,7 +127,10 @@ def request_handler(request_handler):
respond_with_json(
request,
500,
- {"error": "Internal server error"},
+ {
+ "error": "Internal server error",
+ "errcode": Codes.M_UNKNOWN,
+ },
send_cors=True
)
return wrapped_request_handler
--
cgit 1.5.1
From 98dfa7d24f91ff083b36f1379ce2426c8e6cdb75 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 14 Dec 2015 13:55:46 +0000
Subject: Skip events that where the body, name or topic isn't a string when
back populating the FTS index
---
synapse/storage/search.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 39f600f53c..04246101df 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -85,6 +85,11 @@ class SearchStore(BackgroundUpdateStore):
# skip over it.
continue
+ if not isinstance(value, basestring):
+ # If the event body, name or topic isn't a string
+ # then skip over it
+ continue
+
event_search_rows.append((event_id, room_id, key, value))
if isinstance(self.database_engine, PostgresEngine):
--
cgit 1.5.1
From 834924248f4034a209271828d7cca47eee01f328 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 14 Dec 2015 14:09:21 +0000
Subject: Check whether prev_content or prev_sender is set before trying to
rollback state
---
synapse/rest/client/v2_alpha/sync.py | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index f0a637a6da..7cba981c04 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -357,14 +357,19 @@ class SyncRestServlet(RestServlet):
if prev_event_id is None:
del result[event_key]
else:
- result[event_key] = FrozenEvent({
- "type": timeline_event.type,
- "state_key": timeline_event.state_key,
- "content": timeline_event.unsigned['prev_content'],
- "sender": timeline_event.unsigned['prev_sender'],
- "event_id": prev_event_id,
- "room_id": timeline_event.room_id,
- })
+ prev_content = timeline_event.unsigned.get('prev_content')
+ prev_sender = timeline_event.unsigned.get('prev_sender')
+ if prev_content and prev_sender:
+ result[event_key] = FrozenEvent({
+ "type": timeline_event.type,
+ "state_key": timeline_event.state_key,
+ "content": prev_content,
+ "sender": prev_sender,
+ "event_id": prev_event_id,
+ "room_id": timeline_event.room_id,
+ })
+ else:
+ del result[event_key]
logger.debug("New value: %r", result.get(event_key))
return result
--
cgit 1.5.1
From 070e28e203e52fd8968564bec8e73c96c1ab290b Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 14 Dec 2015 14:34:04 +0000
Subject: Combine the prev content tests
---
synapse/rest/client/v2_alpha/sync.py | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 7cba981c04..3f8ce701dc 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -354,22 +354,20 @@ class SyncRestServlet(RestServlet):
logger.debug("Replacing %s with %s in state dict",
timeline_event.event_id, prev_event_id)
- if prev_event_id is None:
+ prev_content = timeline_event.unsigned.get('prev_content')
+ prev_sender = timeline_event.unsigned.get('prev_sender')
+ if prev_event_id is None or not prev_content or not prev_sender:
del result[event_key]
else:
- prev_content = timeline_event.unsigned.get('prev_content')
- prev_sender = timeline_event.unsigned.get('prev_sender')
- if prev_content and prev_sender:
- result[event_key] = FrozenEvent({
- "type": timeline_event.type,
- "state_key": timeline_event.state_key,
- "content": prev_content,
- "sender": prev_sender,
- "event_id": prev_event_id,
- "room_id": timeline_event.room_id,
- })
- else:
- del result[event_key]
+ result[event_key] = FrozenEvent({
+ "type": timeline_event.type,
+ "state_key": timeline_event.state_key,
+ "content": prev_content,
+ "sender": prev_sender,
+ "event_id": prev_event_id,
+ "room_id": timeline_event.room_id,
+ })
+
logger.debug("New value: %r", result.get(event_key))
return result
--
cgit 1.5.1
From 28c5181dfebbce99a4981584a5761285522ed29b Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 14 Dec 2015 14:50:51 +0000
Subject: Add commentary for fix in PR#442
---
synapse/rest/client/v2_alpha/sync.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 3f8ce701dc..adf77e13bf 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -356,6 +356,12 @@ class SyncRestServlet(RestServlet):
prev_content = timeline_event.unsigned.get('prev_content')
prev_sender = timeline_event.unsigned.get('prev_sender')
+ # Empircally it seems possible for the event to have a
+ # "replaces_state" key but not a prev_content or prev_sender
+ # markjh conjectures that it could be due to the server not
+ # having a copy of that event.
+ # If this is the case the we ignore the previous event. This will
+ # cause the displayname calculations on the client to be incorrect
if prev_event_id is None or not prev_content or not prev_sender:
del result[event_key]
else:
--
cgit 1.5.1
From dbe7892e03e2e0e6a50c54109c30b22fe4194894 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 14 Dec 2015 15:09:41 +0000
Subject: Fix a race between started/stopped stream
---
synapse/handlers/events.py | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index fe300433e6..576d77e0e7 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -69,7 +69,12 @@ class EventStreamHandler(BaseHandler):
A deferred that completes once their presence has been updated.
"""
if user not in self._streams_per_user:
- self._streams_per_user[user] = 0
+ # Make sure we set the streams per user to 1 here rather than
+ # setting it to zero and incrementing the value below.
+ # Otherwise this may race with stopped_stream causing the
+ # user to be erased from the map before we have a chance
+ # to increment it.
+ self._streams_per_user[user] = 1
if user in self._stop_timer_per_user:
try:
self.clock.cancel_call_later(
@@ -79,8 +84,8 @@ class EventStreamHandler(BaseHandler):
logger.exception("Failed to cancel event timer")
else:
yield started_user_eventstream(self.distributor, user)
-
- self._streams_per_user[user] += 1
+ else:
+ self._streams_per_user[user] += 1
def stopped_stream(self, user):
"""If there are no streams for a user this starts a timer that will
--
cgit 1.5.1
From 2acae8300fa272caeb774f24d19b80632eca7ae3 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 14 Dec 2015 15:19:37 +0000
Subject: Fix logging to lie less
---
synapse/rest/client/v2_alpha/sync.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index adf77e13bf..b16831246d 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -351,8 +351,6 @@ class SyncRestServlet(RestServlet):
continue
prev_event_id = timeline_event.unsigned.get("replaces_state", None)
- logger.debug("Replacing %s with %s in state dict",
- timeline_event.event_id, prev_event_id)
prev_content = timeline_event.unsigned.get('prev_content')
prev_sender = timeline_event.unsigned.get('prev_sender')
@@ -363,8 +361,17 @@ class SyncRestServlet(RestServlet):
# If this is the case the we ignore the previous event. This will
# cause the displayname calculations on the client to be incorrect
if prev_event_id is None or not prev_content or not prev_sender:
+ logger.debug(
+ "Removing %r from the state dict, as it is missing "
+ " prev_content (prev_event_id=%r)",
+ timeline_event.event_id, prev_event_id
+ )
del result[event_key]
else:
+ logger.debug(
+ "Replacing %r with %r in state dict",
+ timeline_event.event_id, prev_event_id
+ )
result[event_key] = FrozenEvent({
"type": timeline_event.type,
"state_key": timeline_event.state_key,
--
cgit 1.5.1
From 3ddf0b97223fe1f5818251256332c8bad6909020 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 14 Dec 2015 15:20:59 +0000
Subject: Fix spacing
---
synapse/rest/client/v2_alpha/sync.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index b16831246d..73b44e92eb 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -362,7 +362,7 @@ class SyncRestServlet(RestServlet):
# cause the displayname calculations on the client to be incorrect
if prev_event_id is None or not prev_content or not prev_sender:
logger.debug(
- "Removing %r from the state dict, as it is missing "
+ "Removing %r from the state dict, as it is missing"
" prev_content (prev_event_id=%r)",
timeline_event.event_id, prev_event_id
)
--
cgit 1.5.1
From 63fdd9fe0bc5ed3aabe27af7e082d697e4863c83 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Mon, 14 Dec 2015 16:26:59 +0000
Subject: Changelog and version bump for v0.12.0-rc2
---
CHANGES.rst | 11 +++++++++++
synapse/__init__.py | 2 +-
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index f81a51dc7f..6f427f677b 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,14 @@
+Changes in synapse v0.12.0-rc2 (2015-12-14)
+===========================================
+
+* Add caches for whether rooms have been forgotten by a user (PR #434)
+* Remove instructions to use ``--process-dependency-link`` since all of the
+ dependencies of synapse are on PyPI (PR #436)
+* Parallelise the processing of ``/sync`` requests (PR #437)
+* Fix race updating presence in ``/events`` (PR #444)
+* Fix bug back-populating search results (PR #441)
+* Fix bug calculating state in ``/sync`` requests (PR #442)
+
Changes in synapse v0.12.0-rc1 (2015-12-10)
===========================================
diff --git a/synapse/__init__.py b/synapse/__init__.py
index c357f8f9c2..e07c26ccd0 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
""" This is a reference implementation of a Matrix home server.
"""
-__version__ = "0.12.0-rc1"
+__version__ = "0.12.0-rc2"
--
cgit 1.5.1
From dcfc70e8ed263256b2a3cf59e7d21e54f39fc287 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 15 Dec 2015 17:02:21 +0000
Subject: Allow users to change which account a 3pid is bound to
---
synapse/storage/registration.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 2e5eddd259..09a05b08ef 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -258,10 +258,10 @@ class RegistrationStore(SQLBaseStore):
@defer.inlineCallbacks
def user_add_threepid(self, user_id, medium, address, validated_at, added_at):
yield self._simple_upsert("user_threepids", {
- "user_id": user_id,
"medium": medium,
"address": address,
}, {
+ "user_id": user_id,
"validated_at": validated_at,
"added_at": added_at,
})
--
cgit 1.5.1
From 0311612ce9c70d2748cdf2badbd87c854ef5ba8d Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <daniel@matrix.org>
Date: Wed, 16 Dec 2015 13:05:32 +0000
Subject: Give the IS a bunch more 3pid invite context
This allows it to form richer emails
---
synapse/handlers/room.py | 67 +++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 60 insertions(+), 7 deletions(-)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index a72c3fda9f..6a482dacc9 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -704,13 +704,48 @@ class RoomMemberHandler(BaseHandler):
token_id,
txn_id
):
+ room_state = yield self.hs.get_state_handler().get_current_state(room_id)
+
+ inviter_display_name = ""
+ inviter_avatar_url = ""
+ member_event = room_state.get((EventTypes.Member, user.to_string()))
+ if member_event:
+ inviter_display_name = member_event.content.get("displayname", "")
+ inviter_avatar_url = member_event.content.get("avatar_url", "")
+
+ canonical_room_alias = ""
+ canonical_alias_event = room_state.get((EventTypes.CanonicalAlias, ""))
+ if canonical_alias_event:
+ canonical_room_alias = canonical_alias_event.content.get("alias", "")
+
+ room_name = ""
+ room_name_event = room_state.get((EventTypes.Name, ""))
+ if room_name_event:
+ room_name = room_name_event.content.get("name", "")
+
+ room_join_rules = ""
+ join_rules_event = room_state.get((EventTypes.JoinRules, ""))
+ if join_rules_event:
+ room_join_rules = join_rules_event.content.get("join_rule", "")
+
+ room_avatar_url = ""
+ room_avatar_event = room_state.get((EventTypes.RoomAvatar, ""))
+ if room_avatar_event:
+ room_avatar_url = room_avatar_event.content.get("url", "")
+
token, public_key, key_validity_url, display_name = (
yield self._ask_id_server_for_third_party_invite(
- id_server,
- medium,
- address,
- room_id,
- user.to_string()
+ id_server=id_server,
+ medium=medium,
+ address=address,
+ room_id=room_id,
+ inviter_user_id=user.to_string(),
+ room_alias=canonical_room_alias,
+ room_avatar_url=room_avatar_url,
+ room_join_rules=room_join_rules,
+ room_name=room_name,
+ inviter_display_name=inviter_display_name,
+ inviter_avatar_url=inviter_avatar_url
)
)
msg_handler = self.hs.get_handlers().message_handler
@@ -732,7 +767,19 @@ class RoomMemberHandler(BaseHandler):
@defer.inlineCallbacks
def _ask_id_server_for_third_party_invite(
- self, id_server, medium, address, room_id, sender):
+ self,
+ id_server,
+ medium,
+ address,
+ room_id,
+ inviter_user_id,
+ room_alias,
+ room_avatar_url,
+ room_join_rules,
+ room_name,
+ inviter_display_name,
+ inviter_avatar_url
+ ):
is_url = "%s%s/_matrix/identity/api/v1/store-invite" % (
id_server_scheme, id_server,
)
@@ -742,7 +789,13 @@ class RoomMemberHandler(BaseHandler):
"medium": medium,
"address": address,
"room_id": room_id,
- "sender": sender,
+ "room_alias": room_alias,
+ "room_avatar_url": room_avatar_url,
+ "room_join_rules": room_join_rules,
+ "room_name": room_name,
+ "sender": inviter_user_id,
+ "sender_display_name": inviter_display_name,
+ "sender_avatar_url": inviter_avatar_url,
}
)
# TODO: Check for success
--
cgit 1.5.1
From 2b0f8a948286424212e79b9ac5d22d5be6707f1f Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dawagner@gmail.com>
Date: Wed, 16 Dec 2015 17:59:44 +0100
Subject: Fix typo
---
synapse/http/server.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 1b936b6892..682b6b379b 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -129,7 +129,7 @@ def request_handler(request_handler):
500,
{
"error": "Internal server error",
- "errcode": Codes.M_UNKNOWN,
+ "errcode": Codes.UNKNOWN,
},
send_cors=True
)
--
cgit 1.5.1
From a64f9bbfe0fc592043a3da8979b7f2545187dbb6 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 Dec 2015 12:47:26 +0000
Subject: Fix 500 error when back-paginating search results
We were mistakenly adding pagination clauses to the count query, which then
failed because the count query doesn't join to the events table.
---
synapse/storage/search.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 57c9cc1c5f..6cb5e73b6e 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -286,8 +286,10 @@ class SearchStore(BackgroundUpdateStore):
"(%s)" % (" OR ".join(local_clauses),)
)
- count_args = args
- count_clauses = clauses
+ # take copies of the current args and clauses lists, before adding
+ # pagination clauses to main query.
+ count_args = list(args)
+ count_clauses = list(clauses)
if pagination_token:
try:
--
cgit 1.5.1
From 8c5f252edbb0c62663116c6a541ce8691414996a Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dawagner@gmail.com>
Date: Thu, 17 Dec 2015 18:09:51 +0100
Subject: Strip address and such out of 3pid invites
We're not meant to leak that into the graph
---
synapse/api/auth.py | 2 +-
synapse/handlers/federation.py | 13 ++++++++++++-
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index b9c3e6d2c4..adb7d64482 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -778,7 +778,7 @@ class Auth(object):
if "third_party_invite" in event.content:
key = (
EventTypes.ThirdPartyInvite,
- event.content["third_party_invite"]["token"]
+ event.content["third_party_invite"]["signed"]["token"]
)
third_party_invite = current_state.get(key)
if third_party_invite:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index e7ad48c948..1255241461 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1650,11 +1650,22 @@ class FederationHandler(BaseHandler):
sender = invite["sender"]
room_id = invite["room_id"]
+ if "signed" not in invite:
+ logger.info(
+ "Discarding received notification of third party invite "
+ "without signed: %s" % (invite,)
+ )
+ return
+
+ third_party_invite = {
+ "signed": invite["signed"],
+ }
+
event_dict = {
"type": EventTypes.Member,
"content": {
"membership": Membership.INVITE,
- "third_party_invite": invite,
+ "third_party_invite": third_party_invite,
},
"room_id": room_id,
"sender": sender,
--
cgit 1.5.1
From bdacee476d2642753cfa54f5092e56ecb148ff56 Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dawagner@gmail.com>
Date: Thu, 17 Dec 2015 18:31:20 +0100
Subject: Add display_name to 3pid invite in m.room.member invites
---
synapse/handlers/federation.py | 32 +++++++++++++++++++++++++++++++-
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 1255241461..28f2ff68d6 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1650,7 +1650,7 @@ class FederationHandler(BaseHandler):
sender = invite["sender"]
room_id = invite["room_id"]
- if "signed" not in invite:
+ if "signed" not in invite or "token" not in invite["signed"]:
logger.info(
"Discarding received notification of third party invite "
"without signed: %s" % (invite,)
@@ -1676,6 +1676,11 @@ class FederationHandler(BaseHandler):
builder = self.event_builder_factory.new(event_dict)
EventValidator().validate_new(builder)
event, context = yield self._create_new_client_event(builder=builder)
+
+ event, context = yield self.add_display_name_to_third_party_invite(
+ event_dict, event, context
+ )
+
self.auth.check(event, context.current_state)
yield self._validate_keyserver(event, auth_events=context.current_state)
member_handler = self.hs.get_handlers().room_member_handler
@@ -1697,6 +1702,10 @@ class FederationHandler(BaseHandler):
builder=builder,
)
+ event, context = yield self.add_display_name_to_third_party_invite(
+ event_dict, event, context
+ )
+
self.auth.check(event, auth_events=context.current_state)
yield self._validate_keyserver(event, auth_events=context.current_state)
@@ -1706,6 +1715,27 @@ class FederationHandler(BaseHandler):
member_handler = self.hs.get_handlers().room_member_handler
yield member_handler.change_membership(event, context)
+ @defer.inlineCallbacks
+ def add_display_name_to_third_party_invite(self, event_dict, event, context):
+ key = (
+ EventTypes.ThirdPartyInvite,
+ event.content["third_party_invite"]["signed"]["token"]
+ )
+ original_invite = context.current_state.get(key)
+ if not original_invite:
+ logger.info(
+ "Could not find invite event for third_party_invite - "
+ "discarding: %s" % (event_dict,)
+ )
+ return
+
+ display_name = original_invite.content["display_name"]
+ event_dict["content"]["third_party_invite"]["display_name"] = display_name
+ builder = self.event_builder_factory.new(event_dict)
+ EventValidator().validate_new(builder)
+ event, context = yield self._create_new_client_event(builder=builder)
+ defer.returnValue((event, context))
+
@defer.inlineCallbacks
def _validate_keyserver(self, event, auth_events):
token = event.content["third_party_invite"]["signed"]["token"]
--
cgit 1.5.1
From 772ad4f71503866842eb9033b220b757ef20e711 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 17 Dec 2015 23:04:20 +0000
Subject: stop generating default identicons. reverts most of
582019f870adbc4a8a8a9ef97b527e0fead77761 and solves vector-web/vector-im#346
---
synapse/handlers/register.py | 22 +++-------------------
1 file changed, 3 insertions(+), 19 deletions(-)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index a037da0f70..8a365c20f9 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -132,25 +132,9 @@ class RegistrationHandler(BaseHandler):
raise RegistrationError(
500, "Cannot generate user ID.")
- # create a default avatar for the user
- # XXX: ideally clients would explicitly specify one, but given they don't
- # and we want consistent and pretty identicons for random users, we'll
- # do it here.
- try:
- auth_user = UserID.from_string(user_id)
- media_repository = self.hs.get_resource_for_media_repository()
- identicon_resource = media_repository.getChildWithDefault("identicon", None)
- upload_resource = media_repository.getChildWithDefault("upload", None)
- identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320)
- content_uri = yield upload_resource.create_content(
- "image/png", None, identicon_bytes, len(identicon_bytes), auth_user
- )
- profile_handler = self.hs.get_handlers().profile_handler
- profile_handler.set_avatar_url(
- auth_user, auth_user, ("%s#auto" % (content_uri,))
- )
- except NotImplementedError:
- pass # make tests pass without messing around creating default avatars
+ # We used to generate default identicons here, but nowadays
+ # we want clients to generate their own as part of their branding
+ # rather than there being consistent matrix-wide ones, so we don't.
defer.returnValue((user_id, token))
--
cgit 1.5.1
From 64374bda5b47e043a5ff3c0af23bd29461596059 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 17 Dec 2015 23:04:53 +0000
Subject: fix indentation level
---
synapse/handlers/register.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 8a365c20f9..698e7d4479 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -132,9 +132,9 @@ class RegistrationHandler(BaseHandler):
raise RegistrationError(
500, "Cannot generate user ID.")
- # We used to generate default identicons here, but nowadays
- # we want clients to generate their own as part of their branding
- # rather than there being consistent matrix-wide ones, so we don't.
+ # We used to generate default identicons here, but nowadays
+ # we want clients to generate their own as part of their branding
+ # rather than there being consistent matrix-wide ones, so we don't.
defer.returnValue((user_id, token))
--
cgit 1.5.1
From ce4999268a06ccc716d1340b0f4c3e88103d7084 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 18 Dec 2015 10:06:56 +0000
Subject: Fix typo that broke registration on the mobile clients
---
synapse/handlers/register.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index a037da0f70..19df5aa852 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -42,7 +42,7 @@ class RegistrationHandler(BaseHandler):
self.distributor = hs.get_distributor()
self.distributor.declare("registered_user")
- self.captch_client = CaptchaServerHttpClient(hs)
+ self.captcha_client = CaptchaServerHttpClient(hs)
@defer.inlineCallbacks
def check_username(self, localpart):
--
cgit 1.5.1
From 2f871ad143988199a5c3ceac918f721752968e71 Mon Sep 17 00:00:00 2001
From: Mark Haines <mjark@negativecurvature.net>
Date: Fri, 18 Dec 2015 20:44:47 +0000
Subject: Generate code coverage report when running jenkins.sh
---
jenkins.sh | 21 ++++++++++++++++-----
tox.ini | 3 ++-
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/jenkins.sh b/jenkins.sh
index 0018ca610a..7075b1a51a 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -5,9 +5,10 @@ export PYTHONDONTWRITEBYTECODE=yep
# Output test results as junit xml
export TRIAL_FLAGS="--reporter=subunit"
export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
-
-# Output coverage to coverage.xml
-export DUMP_COVERAGE_COMMAND="coverage xml -o coverage.xml"
+# Write coverage reports to a separate file for each process
+# Include branch coverage
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
# Output flake8 violations to violations.flake8.log
# Don't exit with non-0 status code on Jenkins,
@@ -15,6 +16,8 @@ export DUMP_COVERAGE_COMMAND="coverage xml -o coverage.xml"
# UNSTABLE or FAILURE this build.
export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
+rm .coverage.* || echo "No files to remove"
+
tox
: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
@@ -45,7 +48,7 @@ export PERL5LIB PERL_MB_OPT PERL_MM_OPT
: ${PORT_BASE:=8000}
echo >&2 "Running sytest with SQLite3";
-./run-tests.pl -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-sqlite3.tap
+./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-sqlite3.tap
RUN_POSTGRES=""
@@ -64,7 +67,15 @@ done
if test $RUN_POSTGRES = ":$(($PORT_BASE + 1)):$(($PORT_BASE + 2))"; then
echo >&2 "Running sytest with PostgreSQL";
pip install psycopg2
- ./run-tests.pl -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-postgresql.tap
+ ./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-postgresql.tap
else
echo >&2 "Skipping running sytest with PostgreSQL, $RUN_POSTGRES"
fi
+
+cd ..
+cp sytest/.coverage.* .
+
+# Combine the coverage reports
+python -m coverage combine
+# Output coverage to coverage.xml
+coverage xml -o coverage.xml
diff --git a/tox.ini b/tox.ini
index 95424765c3..bd313a4f36 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,7 +11,8 @@ deps =
setenv =
PYTHONDONTWRITEBYTECODE = no_byte_code
commands =
- /bin/bash -c "coverage run --source=synapse {envbindir}/trial {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}"
+ /bin/bash -c "coverage run {env:COVERAGE_OPTS:} --source={toxinidir}/synapse \
+ {envbindir}/trial {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}"
{env:DUMP_COVERAGE_COMMAND:coverage report -m}
[testenv:packaging]
--
cgit 1.5.1
From a6ba41e0785f8f597713bd023e1f6dc3a3d966ea Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Fri, 18 Dec 2015 21:36:42 +0000
Subject: Actually look up required remote server key IDs
set.union() is a side-effect-free function that returns the union of two
sets. This clearly wanted .update(), which is the side-effecting mutator
version.
---
synapse/crypto/keyring.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index bc5bb5cdb1..1fea568eed 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -230,7 +230,9 @@ class Keyring(object):
missing_keys = {}
for group in group_id_to_group.values():
- missing_keys.setdefault(group.server_name, set()).union(group.key_ids)
+ missing_keys.setdefault(group.server_name, set()).update(
+ group.key_ids
+ )
for fn in key_fetch_fns:
results = yield fn(missing_keys.items())
--
cgit 1.5.1
From 64b660682492cd9addaa4a681b0b7780fc23d9d9 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 21 Dec 2015 15:22:03 +0000
Subject: Remove accidentally committed debug logging
---
synapse/rest/client/v2_alpha/sync.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 73b44e92eb..697df03dda 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -104,7 +104,6 @@ class SyncRestServlet(RestServlet):
)
if filter_id and filter_id.startswith('{'):
- logging.error("MJH %r", filter_id)
try:
filter_object = json.loads(filter_id)
except:
--
cgit 1.5.1