From f74e850b5cf7947fbbd13d8bfd1daf43d535741f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Oct 2014 11:46:13 +0100 Subject: remove debugging logging for signing requests --- synapse/http/client.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'synapse') diff --git a/synapse/http/client.py b/synapse/http/client.py index 9f54b74e3a..316ca1ccb9 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -177,10 +177,6 @@ class MatrixHttpClient(BaseHttpClient): request = sign_json(request, self.server_name, self.signing_key) - from syutil.jsonutil import encode_canonical_json - logger.debug("Signing " + " " * 11 + "%s %s", - self.server_name, encode_canonical_json(request)) - auth_headers = [] for key,sig in request["signatures"][self.server_name].items(): -- cgit 1.5.1 From 9aed791fc38790eae6c24e154e7f82ac8509295d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Oct 2014 16:44:27 +0100 Subject: SYN-103: Ignore the 'origin' key in received EDUs. Instead take the origin from the transaction itself --- synapse/federation/replication.py | 2 +- synapse/federation/units.py | 8 ++++++-- tests/federation/test_federation.py | 1 + tests/handlers/test_presence.py | 1 + tests/handlers/test_typing.py | 1 + 5 files changed, 10 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 2346d55045..9363ac7300 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -319,7 +319,7 @@ class ReplicationLayer(object): if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu(edu.origin, edu.edu_type, edu.content) + self.received_edu(transaction.origin, edu.edu_type, edu.content) results = yield defer.DeferredList(dl) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index ecca35ac43..d97aeb698e 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -156,11 +156,15 @@ class Edu(JsonEncodedObject): ] required_keys = [ - "origin", - "destination", "edu_type", ] +# TODO: SYN-103: Remove "origin" and "destination" keys. +# internal_keys = [ +# "origin", +# "destination", +# ] + class Transaction(JsonEncodedObject): """ A transaction is a list of Pdus and Edus to be sent to a remote home diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 8d277d6612..d86ce83b28 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -211,6 +211,7 @@ class FederationTestCase(unittest.TestCase): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" "origin": "test", "destination": "remote", "edu_type": "m.test", diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 15022b8d05..84985a8066 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -43,6 +43,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" keys. "origin": origin, "destination": destination, "edu_type": edu_type, diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 064b04c217..b685373deb 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -33,6 +33,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" keys. "origin": origin, "destination": destination, "edu_type": edu_type, -- cgit 1.5.1 From 13b560971e71a87bf44c35ae9cb4591333dd576c Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 14 Oct 2014 16:47:08 +0100 Subject: Make sure to return an empty JSON object ({}) from presence PUT/POST requests rather than an empty string ("") because most deserialisers won't like the latter --- synapse/rest/presence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index 7fc8ce4404..138cc88a05 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -68,7 +68,7 @@ class PresenceStatusRestServlet(RestServlet): yield self.handlers.presence_handler.set_state( target_user=user, auth_user=auth_user, state=state) - defer.returnValue((200, "")) + defer.returnValue((200, {})) def on_OPTIONS(self, request): return (200, {}) @@ -141,7 +141,7 @@ class PresenceListRestServlet(RestServlet): yield defer.DeferredList(deferreds) - defer.returnValue((200, "")) + defer.returnValue((200, {})) def on_OPTIONS(self, request): return (200, {}) -- cgit 1.5.1 From 456017e0ae6fb542d4cd3bc5977003d556b7bf65 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 16:55:05 +0100 Subject: SPEC-7: Don't stamp event contents with 'hsob_ts' --- synapse/handlers/message.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 317ef2c80c..7b2b8549ed 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -64,7 +64,7 @@ class MessageHandler(BaseHandler): defer.returnValue(None) @defer.inlineCallbacks - def send_message(self, event=None, suppress_auth=False, stamp_event=True): + def send_message(self, event=None, suppress_auth=False): """ Send a message. Args: @@ -72,7 +72,6 @@ class MessageHandler(BaseHandler): suppress_auth (bool) : True to suppress auth for this message. This is primarily so the home server can inject messages into rooms at will. - stamp_event (bool) : True to stamp event content with server keys. Raises: SynapseError if something went wrong. """ @@ -82,9 +81,6 @@ class MessageHandler(BaseHandler): user = self.hs.parse_userid(event.user_id) assert user.is_mine, "User must be our own: %s" % (user,) - if stamp_event: - event.content["hsob_ts"] = int(self.clock.time_msec()) - snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) if not suppress_auth: @@ -132,7 +128,7 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def store_room_data(self, event=None, stamp_event=True): + def store_room_data(self, event=None): """ Stores data for a room. Args: @@ -151,9 +147,6 @@ class MessageHandler(BaseHandler): yield self.auth.check(event, snapshot, raises=True) - if stamp_event: - event.content["hsob_ts"] = int(self.clock.time_msec()) - yield self.state_handler.handle_new_event(event, snapshot) yield self._on_new_room_event(event, snapshot) @@ -221,10 +214,7 @@ class MessageHandler(BaseHandler): defer.returnValue(None) @defer.inlineCallbacks - def send_feedback(self, event, stamp_event=True): - if stamp_event: - event.content["hsob_ts"] = int(self.clock.time_msec()) - + def send_feedback(self, event): snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) yield self.auth.check(event, snapshot, raises=True) -- cgit 1.5.1 From f5cf7ac25b311fda8a2d553f07437b3648c66f6c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 17:12:25 +0100 Subject: SPEC-7: Rename 'ts' to 'origin_server_ts' --- synapse/api/events/factory.py | 4 ++-- synapse/federation/pdu_codec.py | 4 ++-- synapse/federation/persistence.py | 2 +- synapse/federation/replication.py | 4 ++-- synapse/federation/units.py | 16 ++++++++-------- synapse/storage/_base.py | 2 +- synapse/storage/pdu.py | 2 +- synapse/storage/schema/pdu.sql | 2 +- synapse/storage/schema/transactions.sql | 4 ++-- synapse/storage/transactions.py | 14 +++++++------- tests/federation/test_federation.py | 14 +++++++------- tests/federation/test_pdu_codec.py | 4 ++-- tests/handlers/test_federation.py | 6 +++--- tests/handlers/test_presence.py | 2 +- tests/handlers/test_typing.py | 2 +- tests/test_state.py | 2 +- 16 files changed, 42 insertions(+), 42 deletions(-) (limited to 'synapse') diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 0d94850cec..74d0ef77f4 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -58,8 +58,8 @@ class EventFactory(object): random_string(10), self.hs.hostname ) - if "ts" not in kwargs: - kwargs["ts"] = int(self.clock.time_msec()) + if "origin_server_ts" not in kwargs: + kwargs["origin_server_ts"] = int(self.clock.time_msec()) # The "age" key is a delta timestamp that should be converted into an # absolute timestamp the minute we see it. diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index cef61108dd..e8180d94fd 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -96,7 +96,7 @@ class PduCodec(object): if k not in ["event_id", "room_id", "type", "prev_events"] }) - if "ts" not in kwargs: - kwargs["ts"] = int(self.clock.time_msec()) + if "origin_server_ts" not in kwargs: + kwargs["origin_server_ts"] = int(self.clock.time_msec()) return Pdu(**kwargs) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index de36a80e41..7043fcc504 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -157,7 +157,7 @@ class TransactionActions(object): transaction.prev_ids = yield self.store.prep_send_transaction( transaction.transaction_id, transaction.destination, - transaction.ts, + transaction.origin_server_ts, [(p["pdu_id"], p["origin"]) for p in transaction.pdus] ) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9363ac7300..092411eaf9 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -421,7 +421,7 @@ class ReplicationLayer(object): return Transaction( origin=self.server_name, pdus=pdus, - ts=int(self._clock.time_msec()), + origin_server_ts=int(self._clock.time_msec()), destination=None, ) @@ -589,7 +589,7 @@ class _TransactionQueue(object): logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( - ts=self._clock.time_msec(), + origin_server_ts=self._clock.time_msec(), transaction_id=str(self._next_txn_id), origin=self.server_name, destination=destination, diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d97aeb698e..dccac2aca7 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -40,7 +40,7 @@ class Pdu(JsonEncodedObject): { "pdu_id": "78c", - "ts": 1404835423000, + "origin_server_ts": 1404835423000, "origin": "bar", "prev_ids": [ ["23b", "foo"], @@ -55,7 +55,7 @@ class Pdu(JsonEncodedObject): "pdu_id", "context", "origin", - "ts", + "origin_server_ts", "pdu_type", "destinations", "transaction_id", @@ -82,7 +82,7 @@ class Pdu(JsonEncodedObject): "pdu_id", "context", "origin", - "ts", + "origin_server_ts", "pdu_type", "content", ] @@ -186,7 +186,7 @@ class Transaction(JsonEncodedObject): "transaction_id", "origin", "destination", - "ts", + "origin_server_ts", "previous_ids", "pdus", "edus", @@ -203,7 +203,7 @@ class Transaction(JsonEncodedObject): "transaction_id", "origin", "destination", - "ts", + "origin_server_ts", "pdus", ] @@ -225,10 +225,10 @@ class Transaction(JsonEncodedObject): @staticmethod def create_new(pdus, **kwargs): """ Used to create a new transaction. Will auto fill out - transaction_id and ts keys. + transaction_id and origin_server_ts keys. """ - if "ts" not in kwargs: - raise KeyError("Require 'ts' to construct a Transaction") + if "origin_server_ts" not in kwargs: + raise KeyError("Require 'origin_server_ts' to construct a Transaction") if "transaction_id" not in kwargs: raise KeyError( "Require 'transaction_id' to construct a Transaction" diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index dba50f1213..30c5103cdd 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -361,7 +361,7 @@ class SQLBaseStore(object): if "age_ts" not in d: # For compatibility - d["age_ts"] = d["ts"] if "ts" in d else 0 + d["age_ts"] = d["origin_server_ts"] if "origin_server_ts" in d else 0 return self.event_factory.create_event( etype=d["type"], diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index d70467dcd6..61ea979b8a 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -789,7 +789,7 @@ class PdusTable(Table): "origin", "context", "pdu_type", - "ts", + "origin_server_ts", "depth", "is_state", "content_json", diff --git a/synapse/storage/schema/pdu.sql b/synapse/storage/schema/pdu.sql index 16e111a56c..5cc8669912 100644 --- a/synapse/storage/schema/pdu.sql +++ b/synapse/storage/schema/pdu.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS pdus( origin TEXT, context TEXT, pdu_type TEXT, - ts INTEGER, + origin_server_ts INTEGER, depth INTEGER DEFAULT 0 NOT NULL, is_state BOOL, content_json TEXT, diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index 88e3e4e04d..5f8d01327a 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id TEXT, origin TEXT, - ts INTEGER, + origin_server_ts INTEGER, response_code INTEGER, response_json TEXT, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, - ts INTEGER + origin_server_ts INTEGER ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index ab4599b468..a9fa959d49 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -87,7 +87,7 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (code, response_json, transaction_id, origin)) - def prep_send_transaction(self, transaction_id, destination, ts, pdu_list): + def prep_send_transaction(self, transaction_id, destination, origin_server_ts, pdu_list): """Persists an outgoing transaction and calculates the values for the previous transaction id list. @@ -97,7 +97,7 @@ class TransactionStore(SQLBaseStore): Args: transaction_id (str) destination (str) - ts (int) + origin_server_ts (int) pdu_list (list) Returns: @@ -106,10 +106,10 @@ class TransactionStore(SQLBaseStore): return self.runInteraction( self._prep_send_transaction, - transaction_id, destination, ts, pdu_list + transaction_id, destination, origin_server_ts, pdu_list ) - def _prep_send_transaction(self, txn, transaction_id, destination, ts, + def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts, pdu_list): # First we find out what the prev_txs should be. @@ -131,7 +131,7 @@ class TransactionStore(SQLBaseStore): None, transaction_id=transaction_id, destination=destination, - ts=ts, + origin_server_ts=origin_server_ts, response_code=0, response_json=None )) @@ -251,7 +251,7 @@ class ReceivedTransactionsTable(Table): fields = [ "transaction_id", "origin", - "ts", + "origin_server_ts", "response_code", "response_json", "has_been_referenced", @@ -267,7 +267,7 @@ class SentTransactions(Table): "id", "transaction_id", "destination", - "ts", + "origin_server_ts", "response_code", "response_json", ] diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index d86ce83b28..8b1202f6e4 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -99,7 +99,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.topic", - ts=123456789000, + origin_server_ts=123456789000, depth=1, is_state=True, content_json='{"topic":"The topic"}', @@ -134,7 +134,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.text", - ts=123456789001, + origin_server_ts=123456789001, depth=1, content_json='{"text":"Here is the message"}', ) @@ -158,7 +158,7 @@ class FederationTestCase(unittest.TestCase): origin="red", destinations=["remote"], context="my-context", - ts=123456789002, + origin_server_ts=123456789002, pdu_type="m.test", content={"testing": "content here"}, depth=1, @@ -170,14 +170,14 @@ class FederationTestCase(unittest.TestCase): "remote", path="/_matrix/federation/v1/send/1000000/", data={ - "ts": 1000000, + "origin_server_ts": 1000000, "origin": "test", "pdus": [ { "origin": "red", "pdu_id": "abc123def456", "prev_pdus": [], - "ts": 123456789002, + "origin_server_ts": 123456789002, "context": "my-context", "pdu_type": "m.test", "is_state": False, @@ -207,7 +207,7 @@ class FederationTestCase(unittest.TestCase): path="/_matrix/federation/v1/send/1000000/", data={ "origin": "test", - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { @@ -234,7 +234,7 @@ class FederationTestCase(unittest.TestCase): "/_matrix/federation/v1/send/1001000/", """{ "origin": "remote", - "ts": 1001000, + "origin_server_ts": 1001000, "pdus": [], "edus": [ { diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 344e1baf60..0754ef92e8 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -68,7 +68,7 @@ class PduCodecTestCase(unittest.TestCase): context="rooooom", pdu_type="m.room.message", origin="bar.com", - ts=12345, + origin_server_ts=12345, depth=5, prev_pdus=[("alice", "bob.com")], is_state=False, @@ -123,7 +123,7 @@ class PduCodecTestCase(unittest.TestCase): context="rooooom", pdu_type="m.room.topic", origin="bar.com", - ts=12345, + origin_server_ts=12345, depth=5, prev_pdus=[("alice", "bob.com")], is_state=True, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 35c3a4df7b..219b2c4c5e 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -68,7 +68,7 @@ class FederationTestCase(unittest.TestCase): pdu_type=MessageEvent.TYPE, context="foo", content={"msgtype": u"fooo"}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) @@ -95,7 +95,7 @@ class FederationTestCase(unittest.TestCase): target_host=self.hostname, context=room_id, content={}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) @@ -127,7 +127,7 @@ class FederationTestCase(unittest.TestCase): state_key="@red:not%s" % self.hostname, context=room_id, content={}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 84985a8066..1850deacf5 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -39,7 +39,7 @@ ONLINE = PresenceState.ONLINE def _expect_edu(destination, edu_type, content, origin="test"): return { "origin": origin, - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index b685373deb..f1d3b27f74 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -29,7 +29,7 @@ from synapse.handlers.typing import TypingNotificationHandler def _expect_edu(destination, edu_type, content, origin="test"): return { "origin": origin, - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { diff --git a/tests/test_state.py b/tests/test_state.py index b1624f0b25..4b1feaf410 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -599,7 +599,7 @@ def new_fake_pdu(pdu_id, context, pdu_type, state_key, prev_state_id, prev_state_id=prev_state_id, origin="example.com", context="context", - ts=1405353060021, + origin_server_ts=1405353060021, depth=depth, content_json="{}", unrecognized_keys="{}", -- cgit 1.5.1 From 82c582076782f180c9f69a523953c3a36b57b3ac Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 17:31:48 +0100 Subject: keep 'origin_server_ts' as 'ts' in the database to avoid needlessly updating schema --- synapse/federation/units.py | 1 + synapse/storage/__init__.py | 2 ++ synapse/storage/_base.py | 3 ++- synapse/storage/pdu.py | 2 +- synapse/storage/schema/pdu.sql | 2 +- synapse/storage/schema/transactions.sql | 4 ++-- synapse/storage/transactions.py | 13 +++++++------ tests/federation/test_federation.py | 4 ++-- 8 files changed, 18 insertions(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/units.py b/synapse/federation/units.py index dccac2aca7..b2fb964180 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -118,6 +118,7 @@ class Pdu(JsonEncodedObject): """ if pdu_tuple: d = copy.copy(pdu_tuple.pdu_entry._asdict()) + d["origin_server_ts"] = d.pop("ts") d["content"] = json.loads(d["content_json"]) del d["content_json"] diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6dadeb8cce..c8e0efb18f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -155,6 +155,8 @@ class DataStore(RoomMemberStore, RoomStore, cols["unrecognized_keys"] = json.dumps(unrec_keys) + cols["ts"] = cols.pop("origin_server_ts") + logger.debug("Persisting: %s", repr(cols)) if pdu.is_state: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 30c5103cdd..65a86e9056 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -354,6 +354,7 @@ class SQLBaseStore(object): d.pop("stream_ordering", None) d.pop("topological_ordering", None) d.pop("processed", None) + d["origin_server_ts"] = d.pop("ts", 0) d.update(json.loads(row_dict["unrecognized_keys"])) d["content"] = json.loads(d["content"]) @@ -361,7 +362,7 @@ class SQLBaseStore(object): if "age_ts" not in d: # For compatibility - d["age_ts"] = d["origin_server_ts"] if "origin_server_ts" in d else 0 + d["age_ts"] = d.get("origin_server_ts", 0) return self.event_factory.create_event( etype=d["type"], diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 61ea979b8a..d70467dcd6 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -789,7 +789,7 @@ class PdusTable(Table): "origin", "context", "pdu_type", - "origin_server_ts", + "ts", "depth", "is_state", "content_json", diff --git a/synapse/storage/schema/pdu.sql b/synapse/storage/schema/pdu.sql index 5cc8669912..16e111a56c 100644 --- a/synapse/storage/schema/pdu.sql +++ b/synapse/storage/schema/pdu.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS pdus( origin TEXT, context TEXT, pdu_type TEXT, - origin_server_ts INTEGER, + ts INTEGER, depth INTEGER DEFAULT 0 NOT NULL, is_state BOOL, content_json TEXT, diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index 5f8d01327a..88e3e4e04d 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id TEXT, origin TEXT, - origin_server_ts INTEGER, + ts INTEGER, response_code INTEGER, response_json TEXT, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, - origin_server_ts INTEGER + ts INTEGER ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index a9fa959d49..2ba8e30efe 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -87,7 +87,8 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (code, response_json, transaction_id, origin)) - def prep_send_transaction(self, transaction_id, destination, origin_server_ts, pdu_list): + def prep_send_transaction(self, transaction_id, destination, + origin_server_ts, pdu_list): """Persists an outgoing transaction and calculates the values for the previous transaction id list. @@ -109,8 +110,8 @@ class TransactionStore(SQLBaseStore): transaction_id, destination, origin_server_ts, pdu_list ) - def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts, - pdu_list): + def _prep_send_transaction(self, txn, transaction_id, destination, + origin_server_ts, pdu_list): # First we find out what the prev_txs should be. # Since we know that we are only sending one transaction at a time, @@ -131,7 +132,7 @@ class TransactionStore(SQLBaseStore): None, transaction_id=transaction_id, destination=destination, - origin_server_ts=origin_server_ts, + ts=origin_server_ts, response_code=0, response_json=None )) @@ -251,7 +252,7 @@ class ReceivedTransactionsTable(Table): fields = [ "transaction_id", "origin", - "origin_server_ts", + "ts", "response_code", "response_json", "has_been_referenced", @@ -267,7 +268,7 @@ class SentTransactions(Table): "id", "transaction_id", "destination", - "origin_server_ts", + "ts", "response_code", "response_json", ] diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 8b1202f6e4..933aa61c77 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -99,7 +99,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.topic", - origin_server_ts=123456789000, + ts=123456789000, depth=1, is_state=True, content_json='{"topic":"The topic"}', @@ -134,7 +134,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.text", - origin_server_ts=123456789001, + ts=123456789001, depth=1, content_json='{"text":"Here is the message"}', ) -- cgit 1.5.1 From 5662be894e517c0424dcc59127d0c62776510ee7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 20:26:18 +0100 Subject: Bump database version number. --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v6.sql | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/v6.sql (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c8e0efb18f..3aa6345a7f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -64,7 +64,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 5 +SCHEMA_VERSION = 6 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v6.sql b/synapse/storage/schema/delta/v6.sql new file mode 100644 index 0000000000..9bf2068d84 --- /dev/null +++ b/synapse/storage/schema/delta/v6.sql @@ -0,0 +1,31 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS server_tls_certificates( + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. + ts_added_ms INTEGER, -- When the certifcate was added. + tls_certificate BLOB, -- DER encoded x509 certificate. + CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) +); + +CREATE TABLE IF NOT EXISTS server_signature_keys( + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. + ts_added_ms INTEGER, -- When the key was added. + verify_key BLOB, -- NACL verification key. + CONSTRAINT uniqueness UNIQUE (server_name, key_id) +); -- cgit 1.5.1 From 71e6a94af76dbaea592b66c2c065f19f9ef57cb0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 20:26:26 +0100 Subject: Bump version and changelog --- CHANGES.rst | 13 ++++++++++--- UPGRADE.rst | 5 ----- VERSION | 2 +- synapse/__init__.py | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) (limited to 'synapse') diff --git a/CHANGES.rst b/CHANGES.rst index 5b05900daf..dab9285f3b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,13 @@ -Changes in latest -================= -This breaks federation becuase of signing +Changes in synpase 0.4.0 (2014-10-17) +===================================== +This server includes changes to the federation protocol that is not backwards +compatible. + +The Matrix specification has been moved to a seperate git repository. + +Homeserver: + * Sign federation transactions. + * Rename timestamp keys in PDUs. Changes in synapse 0.3.4 (2014-09-25) ===================================== diff --git a/UPGRADE.rst b/UPGRADE.rst index 2ae9254ecf..713fb9ae83 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,8 +1,3 @@ -Upgrading to latest -=================== -This breaks federation between old and new servers due to signing of -transactions. - Upgrading to v0.3.0 =================== diff --git a/VERSION b/VERSION index 42045acae2..1d0ba9ea18 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.4 +0.4.0 diff --git a/synapse/__init__.py b/synapse/__init__.py index a340a5db66..979eac08a7 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.3.4" +__version__ = "0.4.0" -- cgit 1.5.1 From 3187b5ba2db51dc4bac0d20a67f0b6193b45e8cb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 20:56:21 +0100 Subject: add log line for checking verifying signatures --- synapse/crypto/keyring.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 015f76ebe3..2440d604c3 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -38,6 +38,7 @@ class Keyring(object): @defer.inlineCallbacks def verify_json_for_server(self, server_name, json_object): + logger.debug("Verifying for %s", server_name) key_ids = signature_ids(json_object, server_name) if not key_ids: raise SynapseError( -- cgit 1.5.1 From cd198dfea8083132137f6c4df5129fd7bb5f7a1e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 20:58:47 +0100 Subject: More log lines. --- synapse/federation/transport.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 755eee8cf6..81529baee6 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -256,10 +256,14 @@ class TransportLayer(object): def _with_authentication(self, handler): @defer.inlineCallbacks def new_handler(request, *args, **kwargs): - (origin, content) = yield self._authenticate_request(request) - response = yield handler( - origin, content, request.args, *args, **kwargs - ) + try: + (origin, content) = yield self._authenticate_request(request) + response = yield handler( + origin, content, request.args, *args, **kwargs + ) + except: + logger.exception("_authenticate_request failed") + raise defer.returnValue(response) return new_handler @@ -392,9 +396,13 @@ class TransportLayer(object): defer.returnValue((400, {"error": "Invalid transaction"})) return - code, response = yield self.received_handler.on_incoming_transaction( - transaction_data - ) + try: + code, response = yield self.received_handler.on_incoming_transaction( + transaction_data + ) + except: + logger.exception("on_incoming_transaction failed") + raise defer.returnValue((code, response)) -- cgit 1.5.1 From ac9345b47a7c963850369e0a8ad63ed6aaba0795 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 21:00:58 +0100 Subject: Check that we have auth headers and fail nicely --- synapse/federation/transport.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'synapse') diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 81529baee6..e7517cac4d 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -238,6 +238,11 @@ class TransportLayer(object): auth_headers = request.requestHeaders.getRawHeaders(b"Authorization") + if not auth_headers: + raise SynapseError( + 401, "Missing Authorization headers", Codes.UNAUTHORIZED, + ) + for auth in auth_headers: if auth.startswith("X-Matrix"): (origin, key, sig) = parse_auth_header(auth) -- cgit 1.5.1 From b3e6cd59a1c377f81c577e9159c49f6e06cab706 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 18 Oct 2014 00:29:55 +0200 Subject: Bump the other version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 979eac08a7..7067188c5b 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.4.0" +__version__ = "0.4.1" -- cgit 1.5.1 From 9c0826592c8c8dcfe0e3bc0f99bacabd772791f2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Sat, 18 Oct 2014 16:56:44 +0100 Subject: Fix auto generating signing_keys --- synapse/config/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/config/server.py b/synapse/config/server.py index d9d8d0e14e..9332e4acd7 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -94,7 +94,7 @@ class ServerConfig(Config): with open(args.signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, - (syutil.crypto.SigningKey.generate("auto"),), + (syutil.crypto.signing_key.generate_singing_key("auto"),), ) else: signing_keys = cls.read_file(args.signing_key_path, "signing_key") -- cgit 1.5.1 From 4be99c2989799fa4d62682d94aa5b6f90d7750ed Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 20 Oct 2014 14:10:08 +0100 Subject: Add get_json method to 3pid http client. Better logging for errors in 3pid requests --- synapse/handlers/register.py | 3 ++- synapse/http/client.py | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index df562aa762..94b7890b5e 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -15,6 +15,7 @@ """Contains functions for registering clients.""" from twisted.internet import defer +from twisted.python import log from synapse.types import UserID from synapse.api.errors import ( @@ -126,7 +127,7 @@ class RegistrationHandler(BaseHandler): try: threepid = yield self._threepid_from_creds(c) except: - logger.err() + log.err() raise RegistrationError(400, "Couldn't validate 3pid") if not threepid: diff --git a/synapse/http/client.py b/synapse/http/client.py index 316ca1ccb9..46c90dbb76 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -101,7 +101,9 @@ class BaseHttpClient(object): while True: - producer = body_callback(method, url_bytes, headers_dict) + producer = None + if body_callback: + producer = body_callback(method, url_bytes, headers_dict) try: response = yield self.agent.request( @@ -312,6 +314,42 @@ class IdentityServerHttpClient(BaseHttpClient): defer.returnValue(json.loads(body)) + @defer.inlineCallbacks + def get_json(self, destination, path, args={}, retry_on_dns_fail=True): + """ Get's some json from the given host homeserver and path + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + + Returns: + Deferred: Succeeds when we get *any* HTTP response. + + The result of the deferred is a tuple of `(code, response)`, + where `response` is a dict representing the decoded JSON body. + """ + logger.debug("get_json args: %s", args) + + query_bytes = urllib.urlencode(args, True) + logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) + + response = yield self._create_request( + destination.encode("ascii"), + "GET", + path.encode("ascii"), + query_bytes=query_bytes, + retry_on_dns_fail=retry_on_dns_fail, + body_callback=None + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) class CaptchaServerHttpClient(MatrixHttpClient): """Separate HTTP client for talking to google's captcha servers""" -- cgit 1.5.1 From 51edfeb3d0fcde1c578cd6bc74c5b9ee961e9456 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 21 Oct 2014 18:57:13 +0100 Subject: Coturn's timestamps are in seconds, not milliseconds --- synapse/rest/voip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/voip.py b/synapse/rest/voip.py index 2e4627606f..0d0243a249 100644 --- a/synapse/rest/voip.py +++ b/synapse/rest/voip.py @@ -36,7 +36,7 @@ class VoipRestServlet(RestServlet): if not turnUris or not turnSecret or not userLifetime: defer.returnValue( (200, {}) ) - expiry = self.hs.get_clock().time_msec() + userLifetime + expiry = (self.hs.get_clock().time_msec() + userLifetime) / 1000 username = "%d:%s" % (expiry, auth_user.to_string()) mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) -- cgit 1.5.1 From db2e350e294d7b696b47822e6e857f482af2f6f1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Oct 2014 19:04:16 +0100 Subject: Wrap preparing the database in a transaction. Otherwise it will take many seconds to complete because sqlite will create a transaction per statement --- synapse/storage/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 3aa6345a7f..4e9291fdff 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -452,10 +452,11 @@ def prepare_database(db_conn): db_conn.commit() else: + sql_script = "BEGIN TRANSACTION;" for sql_loc in SCHEMAS: - sql_script = read_schema(sql_loc) - - c.executescript(sql_script) + sql_script += read_schema(sql_loc) + sql_script += "COMMIT TRANSACTION;" + c.executescript(sql_script) db_conn.commit() c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) -- cgit 1.5.1 From 15be181642d08337f94b89b49305e49c56a103a4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Oct 2014 19:27:12 +0100 Subject: Add log message if we can't enable ECC. Require pyopenssl>=0.14 since 0.13 doesn't seem to have ECC --- setup.py | 1 + synapse/crypto/context_factory.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/setup.py b/setup.py index 649ac86590..660efd5b89 100755 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ setup( "syutil==0.0.2", "Twisted>=14.0.0", "service_identity>=1.0.0", + "pyopenssl>=0.14", "pyyaml", "pyasn1", "pynacl", diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index f86bd19255..f402c795bb 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -16,6 +16,9 @@ from twisted.internet import ssl from OpenSSL import SSL from twisted.internet._sslverify import _OpenSSLECCurve, _defaultCurveName +import logging + +logger = logging.getLogger(__name__) class ServerContextFactory(ssl.ContextFactory): """Factory for PyOpenSSL SSL contexts that are used to handle incoming @@ -31,7 +34,7 @@ class ServerContextFactory(ssl.ContextFactory): _ecCurve = _OpenSSLECCurve(_defaultCurveName) _ecCurve.addECKeyToContext(context) except: - pass + logger.exception("Failed to enable eliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate(config.tls_certificate) context.use_privatekey(config.tls_private_key) -- cgit 1.5.1