From b95a178584cac07018f47e571f48993878da7284 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 30 Sep 2014 15:15:10 +0100 Subject: SYN-75 Verify signatures on server to server transactions --- synapse/federation/units.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'synapse/federation/units.py') diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 622fe66a8f..1ca123d1bf 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -186,9 +186,6 @@ class Transaction(JsonEncodedObject): "previous_ids", "pdus", "edus", - ] - - internal_keys = [ "transaction_id", "destination", ] -- cgit 1.4.1 From 10ef8e6e4bb9d50fd2c636cfbb66d3dd6d6f94e9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Oct 2014 11:49:40 +0100 Subject: SYN-75 sign at the request level rather than the transaction level --- synapse/federation/replication.py | 13 ---------- synapse/federation/transport.py | 18 +++---------- synapse/federation/units.py | 5 ++++ synapse/http/client.py | 52 ++++++++++++++++++++++++++++++++----- tests/federation/test_federation.py | 4 +-- tests/handlers/test_presence.py | 26 +++++++++---------- tests/handlers/test_typing.py | 4 +-- 7 files changed, 70 insertions(+), 52 deletions(-) (limited to 'synapse/federation/units.py') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index b4235585a3..2346d55045 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -25,8 +25,6 @@ from .persistence import PduActions, TransactionActions from synapse.util.logutils import log_function -from syutil.crypto.jsonsign import sign_json - import logging @@ -66,8 +64,6 @@ class ReplicationLayer(object): hs, self.transaction_actions, transport_layer ) - self.keyring = hs.get_keyring() - self.handler = None self.edu_handlers = {} self.query_handlers = {} @@ -296,10 +292,6 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function def on_incoming_transaction(self, transaction_data): - yield self.keyring.verify_json_for_server( - transaction_data["origin"], transaction_data - ) - transaction = Transaction(**transaction_data) for p in transaction.pdus: @@ -500,7 +492,6 @@ class _TransactionQueue(object): """ def __init__(self, hs, transaction_actions, transport_layer): - self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.transaction_actions = transaction_actions self.transport_layer = transport_layer @@ -615,9 +606,6 @@ class _TransactionQueue(object): # Actually send the transaction - server_name = self.server_name - signing_key = self.signing_key - # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): @@ -627,7 +615,6 @@ class _TransactionQueue(object): for p in data["pdus"]: if "age_ts" in p: p["age"] = now - int(p["age_ts"]) - data = sign_json(data, server_name, signing_key) return data code, response = yield self.transport_layer.send_transaction( diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 1f864f5fa7..48fc9fbf5e 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -163,27 +163,15 @@ class TransportLayer(object): if transaction.destination == self.server_name: raise RuntimeError("Transport layer cannot send to itself!") - if json_data_callback is None: - def json_data_callback(): - return transaction.get_dict() - - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def cb(destination, method, path_bytes, producer): - json_data = json_data_callback() - del json_data["destination"] - del json_data["transaction_id"] - producer.reset(json_data) - + # FIXME: This is only used by the tests. The actual json sent is + # generated by the json_data_callback. json_data = transaction.get_dict() - del json_data["destination"] - del json_data["transaction_id"] code, response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, - on_send_callback=cb, + json_data_callback=json_data_callback, ) logger.debug( diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 1ca123d1bf..ecca35ac43 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -190,6 +190,11 @@ class Transaction(JsonEncodedObject): "destination", ] + internal_keys = [ + "transaction_id", + "destination", + ] + required_keys = [ "transaction_id", "origin", diff --git a/synapse/http/client.py b/synapse/http/client.py index 0e8fa2eb25..62fe14fa5e 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -26,6 +26,8 @@ from syutil.jsonutil import encode_canonical_json from synapse.api.errors import CodeMessageException, SynapseError +from syutil.crypto.jsonsign import sign_json + from StringIO import StringIO import json @@ -147,7 +149,7 @@ class BaseHttpClient(object): class MatrixHttpClient(BaseHttpClient): - """ Wrapper around the twisted HTTP client api. Implements + """ Wrapper around the twisted HTTP client api. Implements Attributes: agent (twisted.web.client.Agent): The twisted Agent used to send the @@ -156,8 +158,38 @@ class MatrixHttpClient(BaseHttpClient): RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" + def __init__(self, hs): + self.signing_key = hs.config.signing_key[0] + self.server_name = hs.hostname + BaseHttpClient.__init__(self, hs) + + def sign_request(self, destination, method, url_bytes, headers_dict, + content=None): + request = { + "method": method, + "uri": url_bytes, + "origin": self.server_name, + "destination": destination, + } + + if content is not None: + request["content"] = content + + request = sign_json(request, self.server_name, self.signing_key) + + auth_headers = [] + + for key,sig in request["signatures"][self.server_name].items(): + auth_headers.append( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + self.server_name, key, sig, + ) + ) + + headers_dict["Authorization"] = auth_headers + @defer.inlineCallbacks - def put_json(self, destination, path, data, on_send_callback=None): + def put_json(self, destination, path, data={}, json_data_callback=None): """ Sends the specifed json data using PUT Args: @@ -166,6 +198,8 @@ class MatrixHttpClient(BaseHttpClient): path (str): The HTTP path. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. + json_data_callback (callable): A callable returning the dict to + use as the request body. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result @@ -173,13 +207,16 @@ class MatrixHttpClient(BaseHttpClient): CodeMessageException is raised. """ - if not on_send_callback: - def on_send_callback(destination, method, path_bytes, producer): - pass + if not json_data_callback: + def json_data_callback(): + return data def body_callback(method, url_bytes, headers_dict): - producer = _JsonProducer(data) - on_send_callback(destination, method, path, producer) + json_data = json_data_callback() + self.sign_request( + destination, method, url_bytes, headers_dict, json_data + ) + producer = _JsonProducer(json_data) return producer response = yield self._create_request( @@ -221,6 +258,7 @@ class MatrixHttpClient(BaseHttpClient): logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) def body_callback(method, url_bytes, headers_dict): + self.sign_request(destination, method, url_bytes, headers_dict) return None response = yield self._create_request( diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 01f07ff36d..91edeaa4b9 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -186,7 +186,7 @@ class FederationTestCase(unittest.TestCase): }, ] }, - on_send_callback=ANY, + json_data_callback=ANY, ) @defer.inlineCallbacks @@ -218,7 +218,7 @@ class FederationTestCase(unittest.TestCase): } ], }, - on_send_callback=ANY, + json_data_callback=ANY, ) @defer.inlineCallbacks diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index c9406b70c4..15022b8d05 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -300,7 +300,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@cabbage:elsewhere", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -329,7 +329,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@apple:test", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -365,7 +365,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@durian:test", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -786,7 +786,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -802,7 +802,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -928,7 +928,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -943,7 +943,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -973,7 +973,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1175,7 +1175,7 @@ class PresencePollingTestCase(unittest.TestCase): "poll": [ "@potato:remote" ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1188,7 +1188,7 @@ class PresencePollingTestCase(unittest.TestCase): "push": [ {"user_id": "@clementine:test" }], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1217,7 +1217,7 @@ class PresencePollingTestCase(unittest.TestCase): "push": [ {"user_id": "@fig:test" }], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1250,7 +1250,7 @@ class PresencePollingTestCase(unittest.TestCase): "unpoll": [ "@potato:remote" ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1282,7 +1282,7 @@ class PresencePollingTestCase(unittest.TestCase): ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index ff40343b8c..064b04c217 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -175,7 +175,7 @@ class TypingNotificationsTestCase(unittest.TestCase): "typing": True, } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -226,7 +226,7 @@ class TypingNotificationsTestCase(unittest.TestCase): "typing": False, } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) -- cgit 1.4.1 From 9aed791fc38790eae6c24e154e7f82ac8509295d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Oct 2014 16:44:27 +0100 Subject: SYN-103: Ignore the 'origin' key in received EDUs. Instead take the origin from the transaction itself --- synapse/federation/replication.py | 2 +- synapse/federation/units.py | 8 ++++++-- tests/federation/test_federation.py | 1 + tests/handlers/test_presence.py | 1 + tests/handlers/test_typing.py | 1 + 5 files changed, 10 insertions(+), 3 deletions(-) (limited to 'synapse/federation/units.py') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 2346d55045..9363ac7300 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -319,7 +319,7 @@ class ReplicationLayer(object): if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu(edu.origin, edu.edu_type, edu.content) + self.received_edu(transaction.origin, edu.edu_type, edu.content) results = yield defer.DeferredList(dl) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index ecca35ac43..d97aeb698e 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -156,11 +156,15 @@ class Edu(JsonEncodedObject): ] required_keys = [ - "origin", - "destination", "edu_type", ] +# TODO: SYN-103: Remove "origin" and "destination" keys. +# internal_keys = [ +# "origin", +# "destination", +# ] + class Transaction(JsonEncodedObject): """ A transaction is a list of Pdus and Edus to be sent to a remote home diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 8d277d6612..d86ce83b28 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -211,6 +211,7 @@ class FederationTestCase(unittest.TestCase): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" "origin": "test", "destination": "remote", "edu_type": "m.test", diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 15022b8d05..84985a8066 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -43,6 +43,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" keys. "origin": origin, "destination": destination, "edu_type": edu_type, diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 064b04c217..b685373deb 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -33,6 +33,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" keys. "origin": origin, "destination": destination, "edu_type": edu_type, -- cgit 1.4.1 From f5cf7ac25b311fda8a2d553f07437b3648c66f6c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 17:12:25 +0100 Subject: SPEC-7: Rename 'ts' to 'origin_server_ts' --- synapse/api/events/factory.py | 4 ++-- synapse/federation/pdu_codec.py | 4 ++-- synapse/federation/persistence.py | 2 +- synapse/federation/replication.py | 4 ++-- synapse/federation/units.py | 16 ++++++++-------- synapse/storage/_base.py | 2 +- synapse/storage/pdu.py | 2 +- synapse/storage/schema/pdu.sql | 2 +- synapse/storage/schema/transactions.sql | 4 ++-- synapse/storage/transactions.py | 14 +++++++------- tests/federation/test_federation.py | 14 +++++++------- tests/federation/test_pdu_codec.py | 4 ++-- tests/handlers/test_federation.py | 6 +++--- tests/handlers/test_presence.py | 2 +- tests/handlers/test_typing.py | 2 +- tests/test_state.py | 2 +- 16 files changed, 42 insertions(+), 42 deletions(-) (limited to 'synapse/federation/units.py') diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 0d94850cec..74d0ef77f4 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -58,8 +58,8 @@ class EventFactory(object): random_string(10), self.hs.hostname ) - if "ts" not in kwargs: - kwargs["ts"] = int(self.clock.time_msec()) + if "origin_server_ts" not in kwargs: + kwargs["origin_server_ts"] = int(self.clock.time_msec()) # The "age" key is a delta timestamp that should be converted into an # absolute timestamp the minute we see it. diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index cef61108dd..e8180d94fd 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -96,7 +96,7 @@ class PduCodec(object): if k not in ["event_id", "room_id", "type", "prev_events"] }) - if "ts" not in kwargs: - kwargs["ts"] = int(self.clock.time_msec()) + if "origin_server_ts" not in kwargs: + kwargs["origin_server_ts"] = int(self.clock.time_msec()) return Pdu(**kwargs) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index de36a80e41..7043fcc504 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -157,7 +157,7 @@ class TransactionActions(object): transaction.prev_ids = yield self.store.prep_send_transaction( transaction.transaction_id, transaction.destination, - transaction.ts, + transaction.origin_server_ts, [(p["pdu_id"], p["origin"]) for p in transaction.pdus] ) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9363ac7300..092411eaf9 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -421,7 +421,7 @@ class ReplicationLayer(object): return Transaction( origin=self.server_name, pdus=pdus, - ts=int(self._clock.time_msec()), + origin_server_ts=int(self._clock.time_msec()), destination=None, ) @@ -589,7 +589,7 @@ class _TransactionQueue(object): logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( - ts=self._clock.time_msec(), + origin_server_ts=self._clock.time_msec(), transaction_id=str(self._next_txn_id), origin=self.server_name, destination=destination, diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d97aeb698e..dccac2aca7 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -40,7 +40,7 @@ class Pdu(JsonEncodedObject): { "pdu_id": "78c", - "ts": 1404835423000, + "origin_server_ts": 1404835423000, "origin": "bar", "prev_ids": [ ["23b", "foo"], @@ -55,7 +55,7 @@ class Pdu(JsonEncodedObject): "pdu_id", "context", "origin", - "ts", + "origin_server_ts", "pdu_type", "destinations", "transaction_id", @@ -82,7 +82,7 @@ class Pdu(JsonEncodedObject): "pdu_id", "context", "origin", - "ts", + "origin_server_ts", "pdu_type", "content", ] @@ -186,7 +186,7 @@ class Transaction(JsonEncodedObject): "transaction_id", "origin", "destination", - "ts", + "origin_server_ts", "previous_ids", "pdus", "edus", @@ -203,7 +203,7 @@ class Transaction(JsonEncodedObject): "transaction_id", "origin", "destination", - "ts", + "origin_server_ts", "pdus", ] @@ -225,10 +225,10 @@ class Transaction(JsonEncodedObject): @staticmethod def create_new(pdus, **kwargs): """ Used to create a new transaction. Will auto fill out - transaction_id and ts keys. + transaction_id and origin_server_ts keys. """ - if "ts" not in kwargs: - raise KeyError("Require 'ts' to construct a Transaction") + if "origin_server_ts" not in kwargs: + raise KeyError("Require 'origin_server_ts' to construct a Transaction") if "transaction_id" not in kwargs: raise KeyError( "Require 'transaction_id' to construct a Transaction" diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index dba50f1213..30c5103cdd 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -361,7 +361,7 @@ class SQLBaseStore(object): if "age_ts" not in d: # For compatibility - d["age_ts"] = d["ts"] if "ts" in d else 0 + d["age_ts"] = d["origin_server_ts"] if "origin_server_ts" in d else 0 return self.event_factory.create_event( etype=d["type"], diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index d70467dcd6..61ea979b8a 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -789,7 +789,7 @@ class PdusTable(Table): "origin", "context", "pdu_type", - "ts", + "origin_server_ts", "depth", "is_state", "content_json", diff --git a/synapse/storage/schema/pdu.sql b/synapse/storage/schema/pdu.sql index 16e111a56c..5cc8669912 100644 --- a/synapse/storage/schema/pdu.sql +++ b/synapse/storage/schema/pdu.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS pdus( origin TEXT, context TEXT, pdu_type TEXT, - ts INTEGER, + origin_server_ts INTEGER, depth INTEGER DEFAULT 0 NOT NULL, is_state BOOL, content_json TEXT, diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index 88e3e4e04d..5f8d01327a 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id TEXT, origin TEXT, - ts INTEGER, + origin_server_ts INTEGER, response_code INTEGER, response_json TEXT, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, - ts INTEGER + origin_server_ts INTEGER ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index ab4599b468..a9fa959d49 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -87,7 +87,7 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (code, response_json, transaction_id, origin)) - def prep_send_transaction(self, transaction_id, destination, ts, pdu_list): + def prep_send_transaction(self, transaction_id, destination, origin_server_ts, pdu_list): """Persists an outgoing transaction and calculates the values for the previous transaction id list. @@ -97,7 +97,7 @@ class TransactionStore(SQLBaseStore): Args: transaction_id (str) destination (str) - ts (int) + origin_server_ts (int) pdu_list (list) Returns: @@ -106,10 +106,10 @@ class TransactionStore(SQLBaseStore): return self.runInteraction( self._prep_send_transaction, - transaction_id, destination, ts, pdu_list + transaction_id, destination, origin_server_ts, pdu_list ) - def _prep_send_transaction(self, txn, transaction_id, destination, ts, + def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts, pdu_list): # First we find out what the prev_txs should be. @@ -131,7 +131,7 @@ class TransactionStore(SQLBaseStore): None, transaction_id=transaction_id, destination=destination, - ts=ts, + origin_server_ts=origin_server_ts, response_code=0, response_json=None )) @@ -251,7 +251,7 @@ class ReceivedTransactionsTable(Table): fields = [ "transaction_id", "origin", - "ts", + "origin_server_ts", "response_code", "response_json", "has_been_referenced", @@ -267,7 +267,7 @@ class SentTransactions(Table): "id", "transaction_id", "destination", - "ts", + "origin_server_ts", "response_code", "response_json", ] diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index d86ce83b28..8b1202f6e4 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -99,7 +99,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.topic", - ts=123456789000, + origin_server_ts=123456789000, depth=1, is_state=True, content_json='{"topic":"The topic"}', @@ -134,7 +134,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.text", - ts=123456789001, + origin_server_ts=123456789001, depth=1, content_json='{"text":"Here is the message"}', ) @@ -158,7 +158,7 @@ class FederationTestCase(unittest.TestCase): origin="red", destinations=["remote"], context="my-context", - ts=123456789002, + origin_server_ts=123456789002, pdu_type="m.test", content={"testing": "content here"}, depth=1, @@ -170,14 +170,14 @@ class FederationTestCase(unittest.TestCase): "remote", path="/_matrix/federation/v1/send/1000000/", data={ - "ts": 1000000, + "origin_server_ts": 1000000, "origin": "test", "pdus": [ { "origin": "red", "pdu_id": "abc123def456", "prev_pdus": [], - "ts": 123456789002, + "origin_server_ts": 123456789002, "context": "my-context", "pdu_type": "m.test", "is_state": False, @@ -207,7 +207,7 @@ class FederationTestCase(unittest.TestCase): path="/_matrix/federation/v1/send/1000000/", data={ "origin": "test", - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { @@ -234,7 +234,7 @@ class FederationTestCase(unittest.TestCase): "/_matrix/federation/v1/send/1001000/", """{ "origin": "remote", - "ts": 1001000, + "origin_server_ts": 1001000, "pdus": [], "edus": [ { diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 344e1baf60..0754ef92e8 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -68,7 +68,7 @@ class PduCodecTestCase(unittest.TestCase): context="rooooom", pdu_type="m.room.message", origin="bar.com", - ts=12345, + origin_server_ts=12345, depth=5, prev_pdus=[("alice", "bob.com")], is_state=False, @@ -123,7 +123,7 @@ class PduCodecTestCase(unittest.TestCase): context="rooooom", pdu_type="m.room.topic", origin="bar.com", - ts=12345, + origin_server_ts=12345, depth=5, prev_pdus=[("alice", "bob.com")], is_state=True, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 35c3a4df7b..219b2c4c5e 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -68,7 +68,7 @@ class FederationTestCase(unittest.TestCase): pdu_type=MessageEvent.TYPE, context="foo", content={"msgtype": u"fooo"}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) @@ -95,7 +95,7 @@ class FederationTestCase(unittest.TestCase): target_host=self.hostname, context=room_id, content={}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) @@ -127,7 +127,7 @@ class FederationTestCase(unittest.TestCase): state_key="@red:not%s" % self.hostname, context=room_id, content={}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 84985a8066..1850deacf5 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -39,7 +39,7 @@ ONLINE = PresenceState.ONLINE def _expect_edu(destination, edu_type, content, origin="test"): return { "origin": origin, - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index b685373deb..f1d3b27f74 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -29,7 +29,7 @@ from synapse.handlers.typing import TypingNotificationHandler def _expect_edu(destination, edu_type, content, origin="test"): return { "origin": origin, - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { diff --git a/tests/test_state.py b/tests/test_state.py index b1624f0b25..4b1feaf410 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -599,7 +599,7 @@ def new_fake_pdu(pdu_id, context, pdu_type, state_key, prev_state_id, prev_state_id=prev_state_id, origin="example.com", context="context", - ts=1405353060021, + origin_server_ts=1405353060021, depth=depth, content_json="{}", unrecognized_keys="{}", -- cgit 1.4.1 From 82c582076782f180c9f69a523953c3a36b57b3ac Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 17:31:48 +0100 Subject: keep 'origin_server_ts' as 'ts' in the database to avoid needlessly updating schema --- synapse/federation/units.py | 1 + synapse/storage/__init__.py | 2 ++ synapse/storage/_base.py | 3 ++- synapse/storage/pdu.py | 2 +- synapse/storage/schema/pdu.sql | 2 +- synapse/storage/schema/transactions.sql | 4 ++-- synapse/storage/transactions.py | 13 +++++++------ tests/federation/test_federation.py | 4 ++-- 8 files changed, 18 insertions(+), 13 deletions(-) (limited to 'synapse/federation/units.py') diff --git a/synapse/federation/units.py b/synapse/federation/units.py index dccac2aca7..b2fb964180 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -118,6 +118,7 @@ class Pdu(JsonEncodedObject): """ if pdu_tuple: d = copy.copy(pdu_tuple.pdu_entry._asdict()) + d["origin_server_ts"] = d.pop("ts") d["content"] = json.loads(d["content_json"]) del d["content_json"] diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6dadeb8cce..c8e0efb18f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -155,6 +155,8 @@ class DataStore(RoomMemberStore, RoomStore, cols["unrecognized_keys"] = json.dumps(unrec_keys) + cols["ts"] = cols.pop("origin_server_ts") + logger.debug("Persisting: %s", repr(cols)) if pdu.is_state: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 30c5103cdd..65a86e9056 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -354,6 +354,7 @@ class SQLBaseStore(object): d.pop("stream_ordering", None) d.pop("topological_ordering", None) d.pop("processed", None) + d["origin_server_ts"] = d.pop("ts", 0) d.update(json.loads(row_dict["unrecognized_keys"])) d["content"] = json.loads(d["content"]) @@ -361,7 +362,7 @@ class SQLBaseStore(object): if "age_ts" not in d: # For compatibility - d["age_ts"] = d["origin_server_ts"] if "origin_server_ts" in d else 0 + d["age_ts"] = d.get("origin_server_ts", 0) return self.event_factory.create_event( etype=d["type"], diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 61ea979b8a..d70467dcd6 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -789,7 +789,7 @@ class PdusTable(Table): "origin", "context", "pdu_type", - "origin_server_ts", + "ts", "depth", "is_state", "content_json", diff --git a/synapse/storage/schema/pdu.sql b/synapse/storage/schema/pdu.sql index 5cc8669912..16e111a56c 100644 --- a/synapse/storage/schema/pdu.sql +++ b/synapse/storage/schema/pdu.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS pdus( origin TEXT, context TEXT, pdu_type TEXT, - origin_server_ts INTEGER, + ts INTEGER, depth INTEGER DEFAULT 0 NOT NULL, is_state BOOL, content_json TEXT, diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index 5f8d01327a..88e3e4e04d 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id TEXT, origin TEXT, - origin_server_ts INTEGER, + ts INTEGER, response_code INTEGER, response_json TEXT, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, - origin_server_ts INTEGER + ts INTEGER ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index a9fa959d49..2ba8e30efe 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -87,7 +87,8 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (code, response_json, transaction_id, origin)) - def prep_send_transaction(self, transaction_id, destination, origin_server_ts, pdu_list): + def prep_send_transaction(self, transaction_id, destination, + origin_server_ts, pdu_list): """Persists an outgoing transaction and calculates the values for the previous transaction id list. @@ -109,8 +110,8 @@ class TransactionStore(SQLBaseStore): transaction_id, destination, origin_server_ts, pdu_list ) - def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts, - pdu_list): + def _prep_send_transaction(self, txn, transaction_id, destination, + origin_server_ts, pdu_list): # First we find out what the prev_txs should be. # Since we know that we are only sending one transaction at a time, @@ -131,7 +132,7 @@ class TransactionStore(SQLBaseStore): None, transaction_id=transaction_id, destination=destination, - origin_server_ts=origin_server_ts, + ts=origin_server_ts, response_code=0, response_json=None )) @@ -251,7 +252,7 @@ class ReceivedTransactionsTable(Table): fields = [ "transaction_id", "origin", - "origin_server_ts", + "ts", "response_code", "response_json", "has_been_referenced", @@ -267,7 +268,7 @@ class SentTransactions(Table): "id", "transaction_id", "destination", - "origin_server_ts", + "ts", "response_code", "response_json", ] diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 8b1202f6e4..933aa61c77 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -99,7 +99,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.topic", - origin_server_ts=123456789000, + ts=123456789000, depth=1, is_state=True, content_json='{"topic":"The topic"}', @@ -134,7 +134,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.text", - origin_server_ts=123456789001, + ts=123456789001, depth=1, content_json='{"text":"Here is the message"}', ) -- cgit 1.4.1