diff options
| -rw-r--r-- | UPGRADE.rst | 4 | ||||
| -rw-r--r-- | scripts/upgrade_appservice_db.py | 54 | ||||
| -rw-r--r-- | synapse/appservice/__init__.py | 49 | ||||
| -rw-r--r-- | synapse/handlers/directory.py | 14 | ||||
| -rw-r--r-- | synapse/handlers/register.py | 11 | ||||
| -rw-r--r-- | synapse/rest/appservice/v1/register.py | 35 | ||||
| -rw-r--r-- | synapse/storage/__init__.py | 2 | ||||
| -rw-r--r-- | synapse/storage/appservice.py | 16 | ||||
| -rw-r--r-- | tests/appservice/test_appservice.py | 87 | ||||
| -rw-r--r-- | tests/storage/test_appservice.py | 12 |
10 files changed, 215 insertions, 69 deletions
diff --git a/UPGRADE.rst b/UPGRADE.rst index 4045baf4e0..f2a042f0e9 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -12,6 +12,10 @@ Servers which use captchas will need to add their public key to:: This is required in order to support registration fallback (typically used on mobile devices). +The format of stored application services has changed in Synapse. You will need +to run ``python upgrade_appservice_db.py <database file path>`` to convert to +the new format. + Upgrading to v0.7.0 =================== diff --git a/scripts/upgrade_appservice_db.py b/scripts/upgrade_appservice_db.py new file mode 100644 index 0000000000..ae1b91c64f --- /dev/null +++ b/scripts/upgrade_appservice_db.py @@ -0,0 +1,54 @@ +from synapse.storage import read_schema +import argparse +import json +import sqlite3 + + +def do_other_deltas(cursor): + cursor.execute("PRAGMA user_version") + row = cursor.fetchone() + + if row and row[0]: + user_version = row[0] + # Run every version since after the current version. + for v in range(user_version + 1, 10): + print "Running delta: %d" % (v,) + sql_script = read_schema("delta/v%d" % (v,)) + cursor.executescript(sql_script) + + +def update_app_service_table(cur): + cur.execute("SELECT id, regex FROM application_services_regex") + for row in cur.fetchall(): + try: + print "checking %s..." % row[0] + json.loads(row[1]) + except ValueError: + # row isn't in json, make it so. + string_regex = row[1] + new_regex = json.dumps({ + "regex": string_regex, + "exclusive": True + }) + cur.execute( + "UPDATE application_services_regex SET regex=? WHERE id=?", + (new_regex, row[0]) + ) + + +def main(dbname): + con = sqlite3.connect(dbname) + cur = con.cursor() + do_other_deltas(cur) + update_app_service_table(cur) + cur.execute("PRAGMA user_version = 14") + cur.close() + con.commit() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("database") + args = parser.parse_args() + + main(args.database) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 381b4cfc4a..a268a6bcc4 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -46,22 +46,34 @@ class ApplicationService(object): def _check_namespaces(self, namespaces): # Sanity check that it is of the form: # { - # users: ["regex",...], - # aliases: ["regex",...], - # rooms: ["regex",...], + # users: [ {regex: "[A-z]+.*", exclusive: true}, ...], + # aliases: [ {regex: "[A-z]+.*", exclusive: true}, ...], + # rooms: [ {regex: "[A-z]+.*", exclusive: true}, ...], # } if not namespaces: return None for ns in ApplicationService.NS_LIST: + if ns not in namespaces: + namespaces[ns] = [] + continue + if type(namespaces[ns]) != list: - raise ValueError("Bad namespace value for '%s'", ns) - for regex in namespaces[ns]: - if not isinstance(regex, basestring): - raise ValueError("Expected string regex for ns '%s'", ns) + raise ValueError("Bad namespace value for '%s'" % ns) + for regex_obj in namespaces[ns]: + if not isinstance(regex_obj, dict): + raise ValueError("Expected dict regex for ns '%s'" % ns) + if not isinstance(regex_obj.get("exclusive"), bool): + raise ValueError( + "Expected bool for 'exclusive' in ns '%s'" % ns + ) + if not isinstance(regex_obj.get("regex"), basestring): + raise ValueError( + "Expected string for 'regex' in ns '%s'" % ns + ) return namespaces - def _matches_regex(self, test_string, namespace_key): + def _matches_regex(self, test_string, namespace_key, return_obj=False): if not isinstance(test_string, basestring): logger.error( "Expected a string to test regex against, but got %s", @@ -69,11 +81,19 @@ class ApplicationService(object): ) return False - for regex in self.namespaces[namespace_key]: - if re.match(regex, test_string): + for regex_obj in self.namespaces[namespace_key]: + if re.match(regex_obj["regex"], test_string): + if return_obj: + return regex_obj return True return False + def _is_exclusive(self, ns_key, test_string): + regex_obj = self._matches_regex(test_string, ns_key, return_obj=True) + if regex_obj: + return regex_obj["exclusive"] + return False + def _matches_user(self, event, member_list): if (hasattr(event, "sender") and self.is_interested_in_user(event.sender)): @@ -143,5 +163,14 @@ class ApplicationService(object): def is_interested_in_room(self, room_id): return self._matches_regex(room_id, ApplicationService.NS_ROOMS) + def is_exclusive_user(self, user_id): + return self._is_exclusive(ApplicationService.NS_USERS, user_id) + + def is_exclusive_alias(self, alias): + return self._is_exclusive(ApplicationService.NS_ALIASES, alias) + + def is_exclusive_room(self, room_id): + return self._is_exclusive(ApplicationService.NS_ROOMS, room_id) + def __str__(self): return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 11d20a5d2d..f76febee8f 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -232,13 +232,23 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def can_modify_alias(self, alias, user_id=None): + # Any application service "interested" in an alias they are regexing on + # can modify the alias. + # Users can only modify the alias if ALL the interested services have + # non-exclusive locks on the alias (or there are no interested services) services = yield self.store.get_app_services() interested_services = [ s for s in services if s.is_interested_in_alias(alias.to_string()) ] + for service in interested_services: if user_id == service.sender: - # this user IS the app service + # this user IS the app service so they can do whatever they like defer.returnValue(True) return - defer.returnValue(len(interested_services) == 0) + elif service.is_exclusive_alias(alias.to_string()): + # another service has an exclusive lock on this alias. + defer.returnValue(False) + return + # either no interested services, or no service with an exclusive lock + defer.returnValue(True) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 516a936cee..cda4a8502a 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -201,11 +201,12 @@ class RegistrationHandler(BaseHandler): interested_services = [ s for s in services if s.is_interested_in_user(user_id) ] - if len(interested_services) > 0: - raise SynapseError( - 400, "This user ID is reserved by an application service.", - errcode=Codes.EXCLUSIVE - ) + for service in interested_services: + if service.is_exclusive_user(user_id): + raise SynapseError( + 400, "This user ID is reserved by an application service.", + errcode=Codes.EXCLUSIVE + ) def _generate_token(self, user_id): # urlsafe variant uses _ and - so use . as the separator and replace diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 3bd0c1220c..a4f6159773 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -48,18 +48,12 @@ class RegisterRestServlet(AppServiceRestServlet): 400, "Missed required keys: as_token(str) / url(str)." ) - namespaces = { - "users": [], - "rooms": [], - "aliases": [] - } - - if "namespaces" in params: - self._parse_namespace(namespaces, params["namespaces"], "users") - self._parse_namespace(namespaces, params["namespaces"], "rooms") - self._parse_namespace(namespaces, params["namespaces"], "aliases") - - app_service = ApplicationService(as_token, as_url, namespaces) + try: + app_service = ApplicationService( + as_token, as_url, params["namespaces"] + ) + except ValueError as e: + raise SynapseError(400, e.message) app_service = yield self.handler.register(app_service) hs_token = app_service.hs_token @@ -68,23 +62,6 @@ class RegisterRestServlet(AppServiceRestServlet): "hs_token": hs_token })) - def _parse_namespace(self, target_ns, origin_ns, ns): - if ns not in target_ns or ns not in origin_ns: - return # nothing to parse / map through to. - - possible_regex_list = origin_ns[ns] - if not type(possible_regex_list) == list: - raise SynapseError(400, "Namespace %s isn't an array." % ns) - - for regex in possible_regex_list: - if not isinstance(regex, basestring): - raise SynapseError( - 400, "Regex '%s' isn't a string in namespace %s" % - (regex, ns) - ) - - target_ns[ns] = origin_ns[ns] - class UnregisterRestServlet(AppServiceRestServlet): """Handles AS registration with the home server. diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d16e7b8fac..3753cd28d0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -74,7 +74,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 13 +SCHEMA_VERSION = 14 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 97481d113b..e30265750a 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import simplejson +from simplejson import JSONDecodeError from twisted.internet import defer from synapse.api.constants import Membership @@ -25,12 +27,18 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) +def log_failure(failure): + logger.error("Failed to detect application services: %s", failure.value) + logger.error(failure.getTraceback()) + + class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.services_cache = [] self.cache_defer = self._populate_cache() + self.cache_defer.addErrback(log_failure) @defer.inlineCallbacks def unregister_app_service(self, token): @@ -130,11 +138,11 @@ class ApplicationServiceStore(SQLBaseStore): ) for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST): if ns_str in service.namespaces: - for regex in service.namespaces[ns_str]: + for regex_obj in service.namespaces[ns_str]: txn.execute( "INSERT INTO application_services_regex(" "as_id, namespace, regex) values(?,?,?)", - (as_id, ns_int, regex) + (as_id, ns_int, simplejson.dumps(regex_obj)) ) return True @@ -311,10 +319,12 @@ class ApplicationServiceStore(SQLBaseStore): try: services[as_token]["namespaces"][ ApplicationService.NS_LIST[ns_int]].append( - res["regex"] + simplejson.loads(res["regex"]) ) except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) + except JSONDecodeError: + logger.error("Bad regex object '%s'", res["regex"]) # TODO get last successful txn id f.e. service for service in services.values(): diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index d12e4f2644..eb7becf725 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -18,6 +18,13 @@ from mock import Mock, PropertyMock from tests import unittest +def _regex(regex, exclusive=True): + return { + "regex": regex, + "exclusive": exclusive + } + + class ApplicationServiceTestCase(unittest.TestCase): def setUp(self): @@ -36,21 +43,21 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_regex_user_id_prefix_match(self): self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@irc_foobar:matrix.org" self.assertTrue(self.service.is_interested(self.event)) def test_regex_user_id_prefix_no_match(self): self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@someone_else:matrix.org" self.assertFalse(self.service.is_interested(self.event)) def test_regex_room_member_is_checked(self): self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@someone_else:matrix.org" self.event.type = "m.room.member" @@ -59,30 +66,78 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_regex_room_id_match(self): self.service.namespaces[ApplicationService.NS_ROOMS].append( - "!some_prefix.*some_suffix:matrix.org" + _regex("!some_prefix.*some_suffix:matrix.org") ) self.event.room_id = "!some_prefixs0m3th1nGsome_suffix:matrix.org" self.assertTrue(self.service.is_interested(self.event)) def test_regex_room_id_no_match(self): self.service.namespaces[ApplicationService.NS_ROOMS].append( - "!some_prefix.*some_suffix:matrix.org" + _regex("!some_prefix.*some_suffix:matrix.org") ) self.event.room_id = "!XqBunHwQIXUiqCaoxq:matrix.org" self.assertFalse(self.service.is_interested(self.event)) def test_regex_alias_match(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#irc_.*:matrix.org" + _regex("#irc_.*:matrix.org") ) self.assertTrue(self.service.is_interested( self.event, aliases_for_event=["#irc_foobar:matrix.org", "#athing:matrix.org"] )) + def test_non_exclusive_alias(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + _regex("#irc_.*:matrix.org", exclusive=False) + ) + self.assertFalse(self.service.is_exclusive_alias( + "#irc_foobar:matrix.org" + )) + + def test_non_exclusive_room(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + _regex("!irc_.*:matrix.org", exclusive=False) + ) + self.assertFalse(self.service.is_exclusive_room( + "!irc_foobar:matrix.org" + )) + + def test_non_exclusive_user(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + _regex("@irc_.*:matrix.org", exclusive=False) + ) + self.assertFalse(self.service.is_exclusive_user( + "@irc_foobar:matrix.org" + )) + + def test_exclusive_alias(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + _regex("#irc_.*:matrix.org", exclusive=True) + ) + self.assertTrue(self.service.is_exclusive_alias( + "#irc_foobar:matrix.org" + )) + + def test_exclusive_user(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + _regex("@irc_.*:matrix.org", exclusive=True) + ) + self.assertTrue(self.service.is_exclusive_user( + "@irc_foobar:matrix.org" + )) + + def test_exclusive_room(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + _regex("!irc_.*:matrix.org", exclusive=True) + ) + self.assertTrue(self.service.is_exclusive_room( + "!irc_foobar:matrix.org" + )) + def test_regex_alias_no_match(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#irc_.*:matrix.org" + _regex("#irc_.*:matrix.org") ) self.assertFalse(self.service.is_interested( self.event, @@ -91,10 +146,10 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_regex_multiple_matches(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#irc_.*:matrix.org" + _regex("#irc_.*:matrix.org") ) self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@irc_foobar:matrix.org" self.assertTrue(self.service.is_interested( @@ -104,10 +159,10 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_restrict_to_rooms(self): self.service.namespaces[ApplicationService.NS_ROOMS].append( - "!flibble_.*:matrix.org" + _regex("!flibble_.*:matrix.org") ) self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@irc_foobar:matrix.org" self.event.room_id = "!wibblewoo:matrix.org" @@ -118,10 +173,10 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_restrict_to_aliases(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#xmpp_.*:matrix.org" + _regex("#xmpp_.*:matrix.org") ) self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@irc_foobar:matrix.org" self.assertFalse(self.service.is_interested( @@ -132,10 +187,10 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_restrict_to_senders(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#xmpp_.*:matrix.org" + _regex("#xmpp_.*:matrix.org") ) self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@xmpp_foobar:matrix.org" self.assertFalse(self.service.is_interested( @@ -146,7 +201,7 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_member_list_match(self): self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) join_list = [ Mock( diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index fc733d4c79..ca5b92ec85 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -50,9 +50,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): def test_update_and_retrieval_of_service(self): url = "https://matrix.org/appservices/foobar" hs_token = "hstok" - user_regex = ["@foobar_.*:matrix.org"] - alias_regex = ["#foobar_.*:matrix.org"] - room_regex = [] + user_regex = [ + {"regex": "@foobar_.*:matrix.org", "exclusive": True} + ] + alias_regex = [ + {"regex": "#foobar_.*:matrix.org", "exclusive": False} + ] + room_regex = [ + + ] service = ApplicationService( url=url, hs_token=hs_token, token=self.as_token, namespaces={ ApplicationService.NS_USERS: user_regex, |