diff --git a/changelog.d/4113.bugfix b/changelog.d/4113.bugfix
new file mode 100644
index 0000000000..7f9d8ee032
--- /dev/null
+++ b/changelog.d/4113.bugfix
@@ -0,0 +1 @@
+Fix e2e key backup with more than 9 backup versions
diff --git a/changelog.d/4163.bugfix b/changelog.d/4163.bugfix
new file mode 100644
index 0000000000..c7e0ee229d
--- /dev/null
+++ b/changelog.d/4163.bugfix
@@ -0,0 +1 @@
+Generating the user consent URI no longer fails on Python 3.
diff --git a/changelog.d/4165.misc b/changelog.d/4165.misc
new file mode 100644
index 0000000000..415d80aeab
--- /dev/null
+++ b/changelog.d/4165.misc
@@ -0,0 +1 @@
+Drop incoming events from federation for unknown rooms
diff --git a/changelog.d/4184.feature b/changelog.d/4184.feature
new file mode 100644
index 0000000000..89975f4c6e
--- /dev/null
+++ b/changelog.d/4184.feature
@@ -0,0 +1 @@
+Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled.
diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py
index 2566ce7cef..e0287c8c6c 100755
--- a/scripts-dev/federation_client.py
+++ b/scripts-dev/federation_client.py
@@ -154,10 +154,15 @@ def request_json(method, origin_name, origin_key, destination, path, content):
s = requests.Session()
s.mount("matrix://", MatrixConnectionAdapter())
+ headers = {"Host": destination, "Authorization": authorization_headers[0]}
+
+ if method == "POST":
+ headers["Content-Type"] = "application/json"
+
result = s.request(
method=method,
url=dest,
- headers={"Host": destination, "Authorization": authorization_headers[0]},
+ headers=headers,
verify=False,
data=content,
)
@@ -203,7 +208,7 @@ def main():
parser.add_argument(
"-X",
"--method",
- help="HTTP method to use for the request. Defaults to GET if --data is"
+ help="HTTP method to use for the request. Defaults to GET if --body is"
"unspecified, POST if it is.",
)
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index fa2cc550e2..98722ae543 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -162,8 +162,30 @@ class FederationServer(FederationBase):
p["age_ts"] = request_time - int(p["age"])
del p["age"]
+ # We try and pull out an event ID so that if later checks fail we
+ # can log something sensible. We don't mandate an event ID here in
+ # case future event formats get rid of the key.
+ possible_event_id = p.get("event_id", "<Unknown>")
+
+ # Now we get the room ID so that we can check that we know the
+ # version of the room.
+ room_id = p.get("room_id")
+ if not room_id:
+ logger.info(
+ "Ignoring PDU as does not have a room_id. Event ID: %s",
+ possible_event_id,
+ )
+ continue
+
+ try:
+ # In future we will actually use the room version to parse the
+ # PDU into an event.
+ yield self.store.get_room_version(room_id)
+ except NotFoundError:
+ logger.info("Ignoring PDU for unknown room_id: %s", room_id)
+ continue
+
event = event_from_pdu_json(p)
- room_id = event.room_id
pdus_by_room.setdefault(room_id, []).append(event)
pdu_results = {}
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 9ca5fd8724..a3bb864bb2 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -202,27 +202,22 @@ class FederationHandler(BaseHandler):
self.room_queues[room_id].append((pdu, origin))
return
- # If we're no longer in the room just ditch the event entirely. This
- # is probably an old server that has come back and thinks we're still
- # in the room (or we've been rejoined to the room by a state reset).
+ # If we're not in the room just ditch the event entirely. This is
+ # probably an old server that has come back and thinks we're still in
+ # the room (or we've been rejoined to the room by a state reset).
#
- # If we were never in the room then maybe our database got vaped and
- # we should check if we *are* in fact in the room. If we are then we
- # can magically rejoin the room.
+ # Note that if we were never in the room then we would have already
+ # dropped the event, since we wouldn't know the room version.
is_in_room = yield self.auth.check_host_in_room(
room_id,
self.server_name
)
if not is_in_room:
- was_in_room = yield self.store.was_host_joined(
- pdu.room_id, self.server_name,
+ logger.info(
+ "[%s %s] Ignoring PDU from %s as we're not in the room",
+ room_id, event_id, origin,
)
- if was_in_room:
- logger.info(
- "[%s %s] Ignoring PDU from %s as we've left the room",
- room_id, event_id, origin,
- )
- defer.returnValue(None)
+ defer.returnValue(None)
state = None
auth_chain = []
@@ -557,86 +552,54 @@ class FederationHandler(BaseHandler):
room_id, event_id, event,
)
- # FIXME (erikj): Awful hack to make the case where we are not currently
- # in the room work
- # If state and auth_chain are None, then we don't need to do this check
- # as we already know we have enough state in the DB to handle this
- # event.
- if state and auth_chain and not event.internal_metadata.is_outlier():
- is_in_room = yield self.auth.check_host_in_room(
- room_id,
- self.server_name
- )
- else:
- is_in_room = True
-
- if not is_in_room:
- logger.info(
- "[%s %s] Got event for room we're not in",
- room_id, event_id,
- )
-
- try:
- yield self._persist_auth_tree(
- origin, auth_chain, state, event
- )
- except AuthError as e:
- raise FederationError(
- "ERROR",
- e.code,
- e.msg,
- affected=event_id,
- )
-
- else:
- event_ids = set()
- if state:
- event_ids |= {e.event_id for e in state}
- if auth_chain:
- event_ids |= {e.event_id for e in auth_chain}
+ event_ids = set()
+ if state:
+ event_ids |= {e.event_id for e in state}
+ if auth_chain:
+ event_ids |= {e.event_id for e in auth_chain}
- seen_ids = yield self.store.have_seen_events(event_ids)
+ seen_ids = yield self.store.have_seen_events(event_ids)
- if state and auth_chain is not None:
- # If we have any state or auth_chain given to us by the replication
- # layer, then we should handle them (if we haven't before.)
+ if state and auth_chain is not None:
+ # If we have any state or auth_chain given to us by the replication
+ # layer, then we should handle them (if we haven't before.)
- event_infos = []
+ event_infos = []
- for e in itertools.chain(auth_chain, state):
- if e.event_id in seen_ids:
- continue
- e.internal_metadata.outlier = True
- auth_ids = e.auth_event_ids()
- auth = {
- (e.type, e.state_key): e for e in auth_chain
- if e.event_id in auth_ids or e.type == EventTypes.Create
- }
- event_infos.append({
- "event": e,
- "auth_events": auth,
- })
- seen_ids.add(e.event_id)
+ for e in itertools.chain(auth_chain, state):
+ if e.event_id in seen_ids:
+ continue
+ e.internal_metadata.outlier = True
+ auth_ids = e.auth_event_ids()
+ auth = {
+ (e.type, e.state_key): e for e in auth_chain
+ if e.event_id in auth_ids or e.type == EventTypes.Create
+ }
+ event_infos.append({
+ "event": e,
+ "auth_events": auth,
+ })
+ seen_ids.add(e.event_id)
- logger.info(
- "[%s %s] persisting newly-received auth/state events %s",
- room_id, event_id, [e["event"].event_id for e in event_infos]
- )
- yield self._handle_new_events(origin, event_infos)
+ logger.info(
+ "[%s %s] persisting newly-received auth/state events %s",
+ room_id, event_id, [e["event"].event_id for e in event_infos]
+ )
+ yield self._handle_new_events(origin, event_infos)
- try:
- context = yield self._handle_new_event(
- origin,
- event,
- state=state,
- )
- except AuthError as e:
- raise FederationError(
- "ERROR",
- e.code,
- e.msg,
- affected=event.event_id,
- )
+ try:
+ context = yield self._handle_new_event(
+ origin,
+ event,
+ state=state,
+ )
+ except AuthError as e:
+ raise FederationError(
+ "ERROR",
+ e.code,
+ e.msg,
+ affected=event.event_id,
+ )
room = yield self.store.get_room(room_id)
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
index a1e4b88e6d..528125e737 100644
--- a/synapse/http/servlet.py
+++ b/synapse/http/servlet.py
@@ -121,16 +121,15 @@ def parse_string(request, name, default=None, required=False,
Args:
request: the twisted HTTP request.
- name (bytes/unicode): the name of the query parameter.
- default (bytes/unicode|None): value to use if the parameter is absent,
+ name (bytes|unicode): the name of the query parameter.
+ default (bytes|unicode|None): value to use if the parameter is absent,
defaults to None. Must be bytes if encoding is None.
required (bool): whether to raise a 400 SynapseError if the
parameter is absent, defaults to False.
- allowed_values (list[bytes/unicode]): List of allowed values for the
+ allowed_values (list[bytes|unicode]): List of allowed values for the
string, or None if any value is allowed, defaults to None. Must be
the same type as name, if given.
- encoding: The encoding to decode the name to, and decode the string
- content with.
+ encoding (str|None): The encoding to decode the string content with.
Returns:
bytes/unicode|None: A string value or the default. Unicode if encoding
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index 8009b7ff1c..ad525b22e1 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -142,10 +142,10 @@ class ConsentResource(Resource):
userhmac = None
has_consented = False
public_version = username == ""
- if not public_version or not self.hs.config.user_consent_at_registration:
- userhmac = parse_string(request, "h", required=True, encoding=None)
+ if not public_version:
+ userhmac_bytes = parse_string(request, "h", required=True, encoding=None)
- self._check_hash(username, userhmac)
+ self._check_hash(username, userhmac_bytes)
if username.startswith('@'):
qualified_user_id = username
@@ -155,15 +155,18 @@ class ConsentResource(Resource):
u = yield self.store.get_user_by_id(qualified_user_id)
if u is None:
raise NotFoundError("Unknown user")
+
has_consented = u["consent_version"] == version
+ userhmac = userhmac_bytes.decode("ascii")
try:
self._render_template(
request, "%s.html" % (version,),
user=username,
- userhmac=userhmac.decode('ascii'),
+ userhmac=userhmac,
version=version,
- has_consented=has_consented, public_version=public_version,
+ has_consented=has_consented,
+ public_version=public_version,
)
except TemplateNotFound:
raise NotFoundError("Unknown policy version")
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index f25ded2295..16b7f005aa 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -118,6 +118,11 @@ class EndToEndRoomKeyStore(SQLBaseStore):
these room keys.
"""
+ try:
+ version = int(version)
+ except ValueError:
+ defer.returnValue({'rooms': {}})
+
keyvalues = {
"user_id": user_id,
"version": version,
@@ -212,14 +217,23 @@ class EndToEndRoomKeyStore(SQLBaseStore):
Raises:
StoreError: with code 404 if there are no e2e_room_keys_versions present
Returns:
- A deferred dict giving the info metadata for this backup version
+ A deferred dict giving the info metadata for this backup version, with
+ fields including:
+ version(str)
+ algorithm(str)
+ auth_data(object): opaque dict supplied by the client
"""
def _get_e2e_room_keys_version_info_txn(txn):
if version is None:
this_version = self._get_current_version(txn, user_id)
else:
- this_version = version
+ try:
+ this_version = int(version)
+ except ValueError:
+ # Our versions are all ints so if we can't convert it to an integer,
+ # it isn't there.
+ raise StoreError(404, "No row found")
result = self._simple_select_one_txn(
txn,
@@ -236,6 +250,7 @@ class EndToEndRoomKeyStore(SQLBaseStore):
),
)
result["auth_data"] = json.loads(result["auth_data"])
+ result["version"] = str(result["version"])
return result
return self.runInteraction(
diff --git a/synapse/storage/schema/delta/52/e2e_room_keys.sql b/synapse/storage/schema/delta/52/e2e_room_keys.sql
new file mode 100644
index 0000000000..db687cccae
--- /dev/null
+++ b/synapse/storage/schema/delta/52/e2e_room_keys.sql
@@ -0,0 +1,53 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Change version column to an integer so we can do MAX() sensibly
+ */
+CREATE TABLE e2e_room_keys_versions_new (
+ user_id TEXT NOT NULL,
+ version BIGINT NOT NULL,
+ algorithm TEXT NOT NULL,
+ auth_data TEXT NOT NULL,
+ deleted SMALLINT DEFAULT 0 NOT NULL
+);
+
+INSERT INTO e2e_room_keys_versions_new
+ SELECT user_id, CAST(version as BIGINT), algorithm, auth_data, deleted FROM e2e_room_keys_versions;
+
+DROP TABLE e2e_room_keys_versions;
+ALTER TABLE e2e_room_keys_versions_new RENAME TO e2e_room_keys_versions;
+
+CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);
+
+/* Change e2e_rooms_keys to match
+ */
+CREATE TABLE e2e_room_keys_new (
+ user_id TEXT NOT NULL,
+ room_id TEXT NOT NULL,
+ session_id TEXT NOT NULL,
+ version BIGINT NOT NULL,
+ first_message_index INT,
+ forwarded_count INT,
+ is_verified BOOLEAN,
+ session_data TEXT NOT NULL
+);
+
+INSERT INTO e2e_room_keys_new
+ SELECT user_id, room_id, session_id, CAST(version as BIGINT), first_message_index, forwarded_count, is_verified, session_data FROM e2e_room_keys;
+
+DROP TABLE e2e_room_keys;
+ALTER TABLE e2e_room_keys_new RENAME TO e2e_room_keys;
+
+CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);
diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py
index df3f1cde6e..4294bbec2a 100644
--- a/tests/rest/client/test_consent.py
+++ b/tests/rest/client/test_consent.py
@@ -60,6 +60,13 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase):
hs = self.setup_test_homeserver(config=config)
return hs
+ def test_render_public_consent(self):
+ """You can observe the terms form without specifying a user"""
+ resource = consent_resource.ConsentResource(self.hs)
+ request, channel = self.make_request("GET", "/consent?v=1", shorthand=False)
+ render(request, resource, self.reactor)
+ self.assertEqual(channel.code, 200)
+
def test_accept_consent(self):
"""
A user can use the consent form to accept the terms.
diff --git a/tests/storage/test_end_to_end_keys.py b/tests/storage/test_end_to_end_keys.py
index 8f0aaece40..b83f7336d3 100644
--- a/tests/storage/test_end_to_end_keys.py
+++ b/tests/storage/test_end_to_end_keys.py
@@ -45,6 +45,21 @@ class EndToEndKeyStoreTestCase(tests.unittest.TestCase):
self.assertDictContainsSubset({"keys": json, "device_display_name": None}, dev)
@defer.inlineCallbacks
+ def test_reupload_key(self):
+ now = 1470174257070
+ json = {"key": "value"}
+
+ yield self.store.store_device("user", "device", None)
+
+ changed = yield self.store.set_e2e_device_keys("user", "device", now, json)
+ self.assertTrue(changed)
+
+ # If we try to upload the same key then we should be told nothing
+ # changed
+ changed = yield self.store.set_e2e_device_keys("user", "device", now, json)
+ self.assertFalse(changed)
+
+ @defer.inlineCallbacks
def test_get_key_with_device_name(self):
now = 1470174257070
json = {"key": "value"}
|
