diff --git a/.circleci/config.yml b/.circleci/config.yml
index ec3848b048..5395028426 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -23,99 +23,106 @@ jobs:
- run: docker push matrixdotorg/synapse:latest
- run: docker push matrixdotorg/synapse:latest-py3
sytestpy2:
- machine: true
+ docker:
+ - image: matrixdotorg/sytest-synapsepy2
+ working_directory: /src
steps:
- checkout
- - run: docker pull matrixdotorg/sytest-synapsepy2
- - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+ - run: /synapse_sytest.sh
- store_artifacts:
- path: ~/project/logs
+ path: /logs
destination: logs
- store_test_results:
- path: logs
+ path: /logs
sytestpy2postgres:
- machine: true
+ docker:
+ - image: matrixdotorg/sytest-synapsepy2
+ working_directory: /src
steps:
- checkout
- - run: docker pull matrixdotorg/sytest-synapsepy2
- - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+ - run: POSTGRES=1 /synapse_sytest.sh
- store_artifacts:
- path: ~/project/logs
+ path: /logs
destination: logs
- store_test_results:
- path: logs
+ path: /logs
sytestpy2merged:
- machine: true
+ docker:
+ - image: matrixdotorg/sytest-synapsepy2
+ working_directory: /src
steps:
- checkout
- run: bash .circleci/merge_base_branch.sh
- - run: docker pull matrixdotorg/sytest-synapsepy2
- - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+ - run: /synapse_sytest.sh
- store_artifacts:
- path: ~/project/logs
+ path: /logs
destination: logs
- store_test_results:
- path: logs
-
+ path: /logs
sytestpy2postgresmerged:
- machine: true
+ docker:
+ - image: matrixdotorg/sytest-synapsepy2
+ working_directory: /src
steps:
- checkout
- run: bash .circleci/merge_base_branch.sh
- - run: docker pull matrixdotorg/sytest-synapsepy2
- - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+ - run: POSTGRES=1 /synapse_sytest.sh
- store_artifacts:
- path: ~/project/logs
+ path: /logs
destination: logs
- store_test_results:
- path: logs
+ path: /logs
sytestpy3:
- machine: true
+ docker:
+ - image: matrixdotorg/sytest-synapsepy3
+ working_directory: /src
steps:
- checkout
- - run: docker pull matrixdotorg/sytest-synapsepy3
- - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
+ - run: /synapse_sytest.sh
- store_artifacts:
- path: ~/project/logs
+ path: /logs
destination: logs
- store_test_results:
- path: logs
+ path: /logs
sytestpy3postgres:
- machine: true
+ docker:
+ - image: matrixdotorg/sytest-synapsepy3
+ working_directory: /src
steps:
- checkout
- - run: docker pull matrixdotorg/sytest-synapsepy3
- - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+ - run: POSTGRES=1 /synapse_sytest.sh
- store_artifacts:
- path: ~/project/logs
+ path: /logs
destination: logs
- store_test_results:
- path: logs
+ path: /logs
sytestpy3merged:
- machine: true
+ docker:
+ - image: matrixdotorg/sytest-synapsepy3
+ working_directory: /src
steps:
- checkout
- run: bash .circleci/merge_base_branch.sh
- - run: docker pull matrixdotorg/sytest-synapsepy3
- - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
+ - run: /synapse_sytest.sh
- store_artifacts:
- path: ~/project/logs
+ path: /logs
destination: logs
- store_test_results:
- path: logs
+ path: /logs
sytestpy3postgresmerged:
- machine: true
+ docker:
+ - image: matrixdotorg/sytest-synapsepy3
+ working_directory: /src
steps:
- checkout
- run: bash .circleci/merge_base_branch.sh
- - run: docker pull matrixdotorg/sytest-synapsepy3
- - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+ - run: POSTGRES=1 /synapse_sytest.sh
- store_artifacts:
- path: ~/project/logs
+ path: /logs
destination: logs
- store_test_results:
- path: logs
+ path: /logs
workflows:
version: 2
diff --git a/.circleci/merge_base_branch.sh b/.circleci/merge_base_branch.sh
index 6b0bf3aa48..b2c8c40f4c 100755
--- a/.circleci/merge_base_branch.sh
+++ b/.circleci/merge_base_branch.sh
@@ -16,7 +16,7 @@ then
GITBASE="develop"
else
# Get the reference, using the GitHub API
- GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+ GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
fi
# Show what we are before
@@ -31,4 +31,4 @@ git fetch -u origin $GITBASE
git merge --no-edit origin/$GITBASE
# Show what we are after.
-git show -s
\ No newline at end of file
+git show -s
diff --git a/CHANGES.md b/CHANGES.md
index 048b9f95db..5f598559a0 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,62 @@
+Synapse 0.33.7 (2018-10-18)
+===========================
+
+**Warning**: This release removes the example email notification templates from
+`res/templates` (they are now internal to the python package). This should only
+affect you if you (a) deploy your Synapse instance from a git checkout or a
+github snapshot URL, and (b) have email notifications enabled.
+
+If you have email notifications enabled, you should ensure that
+`email.template_dir` is either configured to point at a directory where you
+have installed customised templates, or leave it unset to use the default
+templates.
+
+The configuration parser will try to detect the situation where
+`email.template_dir` is incorrectly set to `res/templates` and do the right
+thing, but will warn about this.
+
+Synapse 0.33.7rc2 (2018-10-17)
+==============================
+
+Features
+--------
+
+- Ship the example email templates as part of the package ([\#4052](https://github.com/matrix-org/synapse/issues/4052))
+
+Bugfixes
+--------
+
+- Fix bug which made get_missing_events return too few events ([\#4045](https://github.com/matrix-org/synapse/issues/4045))
+
+
+Synapse 0.33.7rc1 (2018-10-15)
+==============================
+
+Features
+--------
+
+- Add support for end-to-end key backup (MSC1687) ([\#4019](https://github.com/matrix-org/synapse/issues/4019))
+
+
+Bugfixes
+--------
+
+- Fix bug in event persistence logic which caused 'NoneType is not iterable' ([\#3995](https://github.com/matrix-org/synapse/issues/3995))
+- Fix exception in background metrics collection ([\#3996](https://github.com/matrix-org/synapse/issues/3996))
+- Fix exception handling in fetching remote profiles ([\#3997](https://github.com/matrix-org/synapse/issues/3997))
+- Fix handling of rejected threepid invites ([\#3999](https://github.com/matrix-org/synapse/issues/3999))
+- Workers now start on Python 3. ([\#4027](https://github.com/matrix-org/synapse/issues/4027))
+- Synapse now starts on Python 3.7. ([\#4033](https://github.com/matrix-org/synapse/issues/4033))
+
+
+Internal Changes
+----------------
+
+- Log exceptions in looping calls ([\#4008](https://github.com/matrix-org/synapse/issues/4008))
+- Optimisation for serving federation requests ([\#4017](https://github.com/matrix-org/synapse/issues/4017))
+- Add metric to count number of non-empty sync responses ([\#4022](https://github.com/matrix-org/synapse/issues/4022))
+
+
Synapse 0.33.6 (2018-10-04)
===========================
diff --git a/MANIFEST.in b/MANIFEST.in
index c6a37ac685..25cdf0a61b 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -12,12 +12,12 @@ recursive-include synapse/storage/schema *.sql
recursive-include synapse/storage/schema *.py
recursive-include docs *
-recursive-include res *
recursive-include scripts *
recursive-include scripts-dev *
recursive-include synapse *.pyi
recursive-include tests *.py
+recursive-include synapse/res *
recursive-include synapse/static *.css
recursive-include synapse/static *.gif
recursive-include synapse/static *.html
diff --git a/changelog.d/3995.bugfix b/changelog.d/3995.bugfix
deleted file mode 100644
index 2adc36756b..0000000000
--- a/changelog.d/3995.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug in event persistence logic which caused 'NoneType is not iterable'
\ No newline at end of file
diff --git a/changelog.d/3996.bugfix b/changelog.d/3996.bugfix
deleted file mode 100644
index a056485ea1..0000000000
--- a/changelog.d/3996.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix exception in background metrics collection
diff --git a/changelog.d/3997.bugfix b/changelog.d/3997.bugfix
deleted file mode 100644
index b060ee8c18..0000000000
--- a/changelog.d/3997.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix exception handling in fetching remote profiles
diff --git a/changelog.d/3999.bugfix b/changelog.d/3999.bugfix
deleted file mode 100644
index dc3b2caffa..0000000000
--- a/changelog.d/3999.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix handling of rejected threepid invites
diff --git a/changelog.d/4008.misc b/changelog.d/4008.misc
deleted file mode 100644
index 5730210054..0000000000
--- a/changelog.d/4008.misc
+++ /dev/null
@@ -1 +0,0 @@
-Log exceptions in looping calls
diff --git a/changelog.d/4017.misc b/changelog.d/4017.misc
deleted file mode 100644
index b1ceb06560..0000000000
--- a/changelog.d/4017.misc
+++ /dev/null
@@ -1 +0,0 @@
-Optimisation for serving federation requests
\ No newline at end of file
diff --git a/changelog.d/4019.feature b/changelog.d/4019.feature
deleted file mode 100644
index 49e066d269..0000000000
--- a/changelog.d/4019.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for end-to-end key backup (MSC1687)
diff --git a/changelog.d/4022.misc b/changelog.d/4022.misc
deleted file mode 100644
index 5b0e136795..0000000000
--- a/changelog.d/4022.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add metric to count number of non-empty sync responses
diff --git a/changelog.d/4027.bugfix b/changelog.d/4027.bugfix
deleted file mode 100644
index c9bbff3f68..0000000000
--- a/changelog.d/4027.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Workers now start on Python 3.
diff --git a/changelog.d/4031.misc b/changelog.d/4031.misc
new file mode 100644
index 0000000000..60be8b59fd
--- /dev/null
+++ b/changelog.d/4031.misc
@@ -0,0 +1 @@
+Various cleanups in the federation client code
diff --git a/changelog.d/4033.bugfix b/changelog.d/4033.bugfix
deleted file mode 100644
index 4e9e38cdcf..0000000000
--- a/changelog.d/4033.bugfix
+++ /dev/null
@@ -1,2 +0,0 @@
-Synapse now starts on Python 3.7.
-_All_ workers now start on Python 3.
diff --git a/changelog.d/4041.misc b/changelog.d/4041.misc
new file mode 100644
index 0000000000..8cce9daac9
--- /dev/null
+++ b/changelog.d/4041.misc
@@ -0,0 +1 @@
+Run the CircleCI builds in docker containers
diff --git a/changelog.d/4046.bugfix b/changelog.d/4046.bugfix
new file mode 100644
index 0000000000..5046dd1ce3
--- /dev/null
+++ b/changelog.d/4046.bugfix
@@ -0,0 +1 @@
+Fix issue where Python 3 users couldn't paginate /publicRooms
diff --git a/changelog.d/4049.misc b/changelog.d/4049.misc
new file mode 100644
index 0000000000..4370d9dfa6
--- /dev/null
+++ b/changelog.d/4049.misc
@@ -0,0 +1 @@
+Only colourise synctl output when attached to tty
diff --git a/changelog.d/4050.bugfix b/changelog.d/4050.bugfix
new file mode 100644
index 0000000000..3d1f6af847
--- /dev/null
+++ b/changelog.d/4050.bugfix
@@ -0,0 +1 @@
+Fix URL priewing to work in Python 3.7
diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
index cfe88788f2..a38b929f50 100644
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -211,7 +211,9 @@ email:
require_transport_security: False
notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
app_name: Matrix
- template_dir: res/templates
+ # if template_dir is unset, uses the example templates that are part of
+ # the Synapse distribution.
+ #template_dir: res/templates
notif_template_html: notif_mail.html
notif_template_text: notif_mail.txt
notif_for_new_users: True
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 43c5821ade..1ddbbbebfb 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@ try:
except ImportError:
pass
-__version__ = "0.33.6"
+__version__ = "0.33.7"
diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
index fe156b6930..e2582cfecc 100644
--- a/synapse/config/emailconfig.py
+++ b/synapse/config/emailconfig.py
@@ -13,11 +13,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import print_function
+
# This file can't be called email.py because if it is, we cannot:
import email.utils
+import logging
+import os
+import sys
+import textwrap
from ._base import Config
+logger = logging.getLogger(__name__)
+
+TEMPLATE_DIR_WARNING = """\
+WARNING: The email notifier is configured to look for templates in '%(template_dir)s',
+but no templates could be found there. We will fall back to using the example templates;
+to get rid of this warning, leave 'email.template_dir' unset.
+"""
+
class EmailConfig(Config):
def read_config(self, config):
@@ -38,7 +52,6 @@ class EmailConfig(Config):
"smtp_host",
"smtp_port",
"notif_from",
- "template_dir",
"notif_template_html",
"notif_template_text",
]
@@ -62,9 +75,24 @@ class EmailConfig(Config):
self.email_smtp_host = email_config["smtp_host"]
self.email_smtp_port = email_config["smtp_port"]
self.email_notif_from = email_config["notif_from"]
- self.email_template_dir = email_config["template_dir"]
self.email_notif_template_html = email_config["notif_template_html"]
self.email_notif_template_text = email_config["notif_template_text"]
+
+ self.email_template_dir = email_config.get("template_dir")
+
+ # backwards-compatibility hack
+ if (
+ self.email_template_dir == "res/templates"
+ and not os.path.isfile(
+ os.path.join(self.email_template_dir, self.email_notif_template_text)
+ )
+ ):
+ t = TEMPLATE_DIR_WARNING % {
+ "template_dir": self.email_template_dir,
+ }
+ print(textwrap.fill(t, width=80) + "\n", file=sys.stderr)
+ self.email_template_dir = None
+
self.email_notif_for_new_users = email_config.get(
"notif_for_new_users", True
)
@@ -113,7 +141,9 @@ class EmailConfig(Config):
# require_transport_security: False
# notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
# app_name: Matrix
- # template_dir: res/templates
+ # # if template_dir is unset, uses the example templates that are part of
+ # # the Synapse distribution.
+ # #template_dir: res/templates
# notif_template_html: notif_mail.html
# notif_template_text: notif_mail.txt
# notif_for_new_users: True
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 819e8f7331..4efe95faa4 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -507,19 +507,19 @@ class FederationServer(FederationBase):
@defer.inlineCallbacks
@log_function
def on_get_missing_events(self, origin, room_id, earliest_events,
- latest_events, limit, min_depth):
+ latest_events, limit):
with (yield self._server_linearizer.queue((origin, room_id))):
origin_host, _ = parse_server_name(origin)
yield self.check_server_matches_acl(origin_host, room_id)
logger.info(
"on_get_missing_events: earliest_events: %r, latest_events: %r,"
- " limit: %d, min_depth: %d",
- earliest_events, latest_events, limit, min_depth
+ " limit: %d",
+ earliest_events, latest_events, limit,
)
missing_events = yield self.handler.on_get_missing_events(
- origin, room_id, earliest_events, latest_events, limit, min_depth
+ origin, room_id, earliest_events, latest_events, limit,
)
if len(missing_events) < 5:
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 98b5950800..3fdd63be95 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -633,14 +633,6 @@ class TransactionQueue(object):
transaction, json_data_cb
)
code = 200
-
- if response:
- for e_id, r in response.get("pdus", {}).items():
- if "error" in r:
- logger.warn(
- "Transaction returned error for %s: %s",
- e_id, r,
- )
except HttpResponseException as e:
code = e.code
response = e.response
@@ -657,19 +649,24 @@ class TransactionQueue(object):
destination, txn_id, code
)
- logger.debug("TX [%s] Sent transaction", destination)
- logger.debug("TX [%s] Marking as delivered...", destination)
-
yield self.transaction_actions.delivered(
transaction, code, response
)
- logger.debug("TX [%s] Marked as delivered", destination)
+ logger.debug("TX [%s] {%s} Marked as delivered", destination, txn_id)
- if code != 200:
+ if code == 200:
+ for e_id, r in response.get("pdus", {}).items():
+ if "error" in r:
+ logger.warn(
+ "TX [%s] {%s} Remote returned error for %s: %s",
+ destination, txn_id, e_id, r,
+ )
+ else:
for p in pdus:
- logger.info(
- "Failed to send event %s to %s", p.event_id, destination
+ logger.warn(
+ "TX [%s] {%s} Failed to send event %s",
+ destination, txn_id, p.event_id,
)
success = False
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 2ab973d6c8..edba5a9808 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -143,9 +143,17 @@ class TransportLayerClient(object):
transaction (Transaction)
Returns:
- Deferred: Results of the deferred is a tuple in the form of
- (response_code, response_body) where the response_body is a
- python dict decoded from json
+ Deferred: Succeeds when we get a 2xx HTTP response. The result
+ will be the decoded JSON body.
+
+ Fails with ``HTTPRequestException`` if we get an HTTP response
+ code >= 300.
+
+ Fails with ``NotRetryingDestination`` if we are not yet ready
+ to retry this server.
+
+ Fails with ``FederationDeniedError`` if this destination
+ is not on our federation whitelist
"""
logger.debug(
"send_data dest=%s, txid=%s",
@@ -170,11 +178,6 @@ class TransportLayerClient(object):
backoff_on_404=True, # If we get a 404 the other side has gone
)
- logger.debug(
- "send_data dest=%s, txid=%s, got response: 200",
- transaction.destination, transaction.transaction_id,
- )
-
defer.returnValue(response)
@defer.inlineCallbacks
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 2f874b4838..7288d49074 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -560,7 +560,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet):
@defer.inlineCallbacks
def on_POST(self, origin, content, query, room_id):
limit = int(content.get("limit", 10))
- min_depth = int(content.get("min_depth", 0))
earliest_events = content.get("earliest_events", [])
latest_events = content.get("latest_events", [])
@@ -569,7 +568,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet):
room_id=room_id,
earliest_events=earliest_events,
latest_events=latest_events,
- min_depth=min_depth,
limit=limit,
)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 45d955e6f5..cab57a8849 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -309,8 +309,8 @@ class FederationHandler(BaseHandler):
if sent_to_us_directly:
logger.warn(
- "[%s %s] Failed to fetch %d prev events: rejecting",
- room_id, event_id, len(prevs - seen),
+ "[%s %s] Rejecting: failed to fetch %d prev events: %s",
+ room_id, event_id, len(prevs - seen), shortstr(prevs - seen)
)
raise FederationError(
"ERROR",
@@ -452,8 +452,8 @@ class FederationHandler(BaseHandler):
latest |= seen
logger.info(
- "[%s %s]: Requesting %d prev_events: %s",
- room_id, event_id, len(prevs - seen), shortstr(prevs - seen)
+ "[%s %s]: Requesting missing events between %s and %s",
+ room_id, event_id, shortstr(latest), event_id,
)
# XXX: we set timeout to 10s to help workaround
@@ -1852,7 +1852,7 @@ class FederationHandler(BaseHandler):
@defer.inlineCallbacks
def on_get_missing_events(self, origin, room_id, earliest_events,
- latest_events, limit, min_depth):
+ latest_events, limit):
in_room = yield self.auth.check_host_in_room(
room_id,
origin
@@ -1861,14 +1861,12 @@ class FederationHandler(BaseHandler):
raise AuthError(403, "Host not in room.")
limit = min(limit, 20)
- min_depth = max(min_depth, 0)
missing_events = yield self.store.get_missing_events(
room_id=room_id,
earliest_events=earliest_events,
latest_events=latest_events,
limit=limit,
- min_depth=min_depth,
)
missing_events = yield filter_events_for_server(
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 38e1737ec9..dc88620885 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -16,7 +16,7 @@
import logging
from collections import namedtuple
-from six import iteritems
+from six import PY3, iteritems
from six.moves import range
import msgpack
@@ -444,9 +444,16 @@ class RoomListNextBatch(namedtuple("RoomListNextBatch", (
@classmethod
def from_token(cls, token):
+ if PY3:
+ # The argument raw=False is only available on new versions of
+ # msgpack, and only really needed on Python 3. Gate it behind
+ # a PY3 check to avoid causing issues on Debian-packaged versions.
+ decoded = msgpack.loads(decode_base64(token), raw=False)
+ else:
+ decoded = msgpack.loads(decode_base64(token))
return RoomListNextBatch(**{
cls.REVERSE_KEY_DICT[key]: val
- for key, val in msgpack.loads(decode_base64(token)).items()
+ for key, val in decoded.items()
})
def to_token(self):
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 14b12cd1c4..fcc02fc77d 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -195,7 +195,7 @@ class MatrixFederationHttpClient(object):
)
self.clock = hs.get_clock()
self._store = hs.get_datastore()
- self.version_string = hs.version_string.encode('ascii')
+ self.version_string_bytes = hs.version_string.encode('ascii')
self.default_timeout = 60
def schedule(x):
@@ -261,8 +261,8 @@ class MatrixFederationHttpClient(object):
ignore_backoff=ignore_backoff,
)
- method = request.method
- destination = request.destination
+ method_bytes = request.method.encode("ascii")
+ destination_bytes = request.destination.encode("ascii")
path_bytes = request.path.encode("ascii")
if request.query:
query_bytes = encode_query_args(request.query)
@@ -270,8 +270,8 @@ class MatrixFederationHttpClient(object):
query_bytes = b""
headers_dict = {
- "User-Agent": [self.version_string],
- "Host": [request.destination],
+ b"User-Agent": [self.version_string_bytes],
+ b"Host": [destination_bytes],
}
with limiter:
@@ -282,50 +282,51 @@ class MatrixFederationHttpClient(object):
else:
retries_left = MAX_SHORT_RETRIES
- url = urllib.parse.urlunparse((
- b"matrix", destination.encode("ascii"),
+ url_bytes = urllib.parse.urlunparse((
+ b"matrix", destination_bytes,
path_bytes, None, query_bytes, b"",
- )).decode('ascii')
+ ))
+ url_str = url_bytes.decode('ascii')
- http_url = urllib.parse.urlunparse((
+ url_to_sign_bytes = urllib.parse.urlunparse((
b"", b"",
path_bytes, None, query_bytes, b"",
- )).decode('ascii')
+ ))
while True:
try:
json = request.get_json()
if json:
- data = encode_canonical_json(json)
- headers_dict["Content-Type"] = ["application/json"]
+ headers_dict[b"Content-Type"] = [b"application/json"]
self.sign_request(
- destination, method, http_url, headers_dict, json
+ destination_bytes, method_bytes, url_to_sign_bytes,
+ headers_dict, json,
)
- else:
- data = None
- self.sign_request(destination, method, http_url, headers_dict)
-
- logger.info(
- "{%s} [%s] Sending request: %s %s",
- request.txn_id, destination, method, url
- )
-
- if data:
+ data = encode_canonical_json(json)
producer = FileBodyProducer(
BytesIO(data),
- cooperator=self._cooperator
+ cooperator=self._cooperator,
)
else:
producer = None
+ self.sign_request(
+ destination_bytes, method_bytes, url_to_sign_bytes,
+ headers_dict,
+ )
- request_deferred = treq.request(
- method,
- url,
+ logger.info(
+ "{%s} [%s] Sending request: %s %s",
+ request.txn_id, request.destination, request.method,
+ url_str,
+ )
+
+ # we don't want all the fancy cookie and redirect handling that
+ # treq.request gives: just use the raw Agent.
+ request_deferred = self.agent.request(
+ method_bytes,
+ url_bytes,
headers=Headers(headers_dict),
- data=producer,
- agent=self.agent,
- reactor=self.hs.get_reactor(),
- unbuffered=True
+ bodyProducer=producer,
)
request_deferred = timeout_deferred(
@@ -344,9 +345,9 @@ class MatrixFederationHttpClient(object):
logger.warn(
"{%s} [%s] Request failed: %s %s: %s",
request.txn_id,
- destination,
- method,
- url,
+ request.destination,
+ request.method,
+ url_str,
_flatten_response_never_received(e),
)
@@ -366,7 +367,7 @@ class MatrixFederationHttpClient(object):
logger.debug(
"{%s} [%s] Waiting %ss before re-sending...",
request.txn_id,
- destination,
+ request.destination,
delay,
)
@@ -378,7 +379,7 @@ class MatrixFederationHttpClient(object):
logger.info(
"{%s} [%s] Got response headers: %d %s",
request.txn_id,
- destination,
+ request.destination,
response.code,
response.phrase.decode('ascii', errors='replace'),
)
@@ -411,8 +412,9 @@ class MatrixFederationHttpClient(object):
destination_is must be non-None.
method (bytes): The HTTP method of the request
url_bytes (bytes): The URI path of the request
- headers_dict (dict): Dictionary of request headers to append to
- content (bytes): The body of the request
+ headers_dict (dict[bytes, list[bytes]]): Dictionary of request headers to
+ append to
+ content (object): The body of the request
destination_is (bytes): As 'destination', but if the destination is an
identity server
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 1a5a10d974..b9dcfee740 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -528,7 +528,10 @@ def load_jinja2_templates(config):
"""
logger.info("loading jinja2")
- loader = jinja2.FileSystemLoader(config.email_template_dir)
+ if config.email_template_dir:
+ loader = jinja2.FileSystemLoader(config.email_template_dir)
+ else:
+ loader = jinja2.PackageLoader('synapse', 'res/templates')
env = jinja2.Environment(loader=loader)
env.filters["format_ts"] = format_ts_filter
env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config)
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 2947f37f1a..f51184b50d 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -55,7 +55,7 @@ REQUIREMENTS = {
"sortedcontainers>=1.4.4": ["sortedcontainers"],
"pysaml2>=3.0.0": ["saml2"],
"pymacaroons-pynacl>=0.9.3": ["pymacaroons"],
- "msgpack-python>=0.3.0": ["msgpack"],
+ "msgpack-python>=0.4.2": ["msgpack"],
"phonenumbers>=8.2.0": ["phonenumbers"],
"six>=1.10": ["six"],
diff --git a/res/templates/mail-Vector.css b/synapse/res/templates/mail-Vector.css
index 6a3e36eda1..6a3e36eda1 100644
--- a/res/templates/mail-Vector.css
+++ b/synapse/res/templates/mail-Vector.css
diff --git a/res/templates/mail.css b/synapse/res/templates/mail.css
index 5ab3e1b06d..5ab3e1b06d 100644
--- a/res/templates/mail.css
+++ b/synapse/res/templates/mail.css
diff --git a/res/templates/notif.html b/synapse/res/templates/notif.html
index 88b921ca9c..88b921ca9c 100644
--- a/res/templates/notif.html
+++ b/synapse/res/templates/notif.html
diff --git a/res/templates/notif.txt b/synapse/res/templates/notif.txt
index a37bee9833..a37bee9833 100644
--- a/res/templates/notif.txt
+++ b/synapse/res/templates/notif.txt
diff --git a/res/templates/notif_mail.html b/synapse/res/templates/notif_mail.html
index fcdb3109fe..fcdb3109fe 100644
--- a/res/templates/notif_mail.html
+++ b/synapse/res/templates/notif_mail.html
diff --git a/res/templates/notif_mail.txt b/synapse/res/templates/notif_mail.txt
index 24843042a5..24843042a5 100644
--- a/res/templates/notif_mail.txt
+++ b/synapse/res/templates/notif_mail.txt
diff --git a/res/templates/room.html b/synapse/res/templates/room.html
index 723c222d25..723c222d25 100644
--- a/res/templates/room.html
+++ b/synapse/res/templates/room.html
diff --git a/res/templates/room.txt b/synapse/res/templates/room.txt
index 84648c710e..84648c710e 100644
--- a/res/templates/room.txt
+++ b/synapse/res/templates/room.txt
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index af01040a38..8c892ff187 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -596,10 +596,13 @@ def _iterate_over_text(tree, *tags_to_ignore):
# to be returned.
elements = iter([tree])
while True:
- el = next(elements)
+ el = next(elements, None)
+ if el is None:
+ return
+
if isinstance(el, string_types):
yield el
- elif el is not None and el.tag not in tags_to_ignore:
+ elif el.tag not in tags_to_ignore:
# el.text is the text before the first child, so we can immediately
# return it if the text exists.
if el.text:
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 24345b20a6..3faca2a042 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -376,33 +376,25 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore,
@defer.inlineCallbacks
def get_missing_events(self, room_id, earliest_events, latest_events,
- limit, min_depth):
+ limit):
ids = yield self.runInteraction(
"get_missing_events",
self._get_missing_events,
- room_id, earliest_events, latest_events, limit, min_depth
+ room_id, earliest_events, latest_events, limit,
)
-
events = yield self._get_events(ids)
-
- events = sorted(
- [ev for ev in events if ev.depth >= min_depth],
- key=lambda e: e.depth,
- )
-
- defer.returnValue(events[:limit])
+ defer.returnValue(events)
def _get_missing_events(self, txn, room_id, earliest_events, latest_events,
- limit, min_depth):
-
- earliest_events = set(earliest_events)
- front = set(latest_events) - earliest_events
+ limit):
- event_results = set()
+ seen_events = set(earliest_events)
+ front = set(latest_events) - seen_events
+ event_results = []
query = (
"SELECT prev_event_id FROM event_edges "
- "WHERE event_id = ? AND is_state = ? "
+ "WHERE room_id = ? AND event_id = ? AND is_state = ? "
"LIMIT ?"
)
@@ -411,18 +403,20 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore,
for event_id in front:
txn.execute(
query,
- (event_id, False, limit - len(event_results))
+ (room_id, event_id, False, limit - len(event_results))
)
- for e_id, in txn:
- new_front.add(e_id)
+ new_results = set(t[0] for t in txn) - seen_events
- new_front -= earliest_events
- new_front -= event_results
+ new_front |= new_results
+ seen_events |= new_results
+ event_results.extend(new_results)
front = new_front
- event_results |= new_front
+ # we built the list working backwards from latest_events; we now need to
+ # reverse it so that the events are approximately chronological.
+ event_results.reverse()
return event_results
diff --git a/synctl b/synctl
index 09b64459b1..4bd4c68b37 100755
--- a/synctl
+++ b/synctl
@@ -48,7 +48,16 @@ def pid_running(pid):
def write(message, colour=NORMAL, stream=sys.stdout):
- if colour == NORMAL:
+ # Lets check if we're writing to a TTY before colouring
+ should_colour = False
+ try:
+ should_colour = stream.isatty()
+ except AttributeError:
+ # Just in case `isatty` isn't defined on everything. The python
+ # docs are incredibly vague.
+ pass
+
+ if not should_colour:
stream.write(message + "\n")
else:
stream.write(colour + message + NORMAL + "\n")
diff --git a/tests/handlers/test_roomlist.py b/tests/handlers/test_roomlist.py
new file mode 100644
index 0000000000..61eebb6985
--- /dev/null
+++ b/tests/handlers/test_roomlist.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.handlers.room_list import RoomListNextBatch
+
+import tests.unittest
+import tests.utils
+
+
+class RoomListTestCase(tests.unittest.TestCase):
+ """ Tests RoomList's RoomListNextBatch. """
+
+ def setUp(self):
+ pass
+
+ def test_check_read_batch_tokens(self):
+ batch_token = RoomListNextBatch(
+ stream_ordering="abcdef",
+ public_room_stream_id="123",
+ current_limit=20,
+ direction_is_forward=True,
+ ).to_token()
+ next_batch = RoomListNextBatch.from_token(batch_token)
+ self.assertEquals(next_batch.stream_ordering, "abcdef")
+ self.assertEquals(next_batch.public_room_stream_id, "123")
+ self.assertEquals(next_batch.current_limit, 20)
+ self.assertEquals(next_batch.direction_is_forward, True)
|
