diff --git a/UPGRADE.rst b/UPGRADE.rst
index 4045baf4e0..f2a042f0e9 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -12,6 +12,10 @@ Servers which use captchas will need to add their public key to::
This is required in order to support registration fallback (typically used on
mobile devices).
+The format of stored application services has changed in Synapse. You will need
+to run ``python upgrade_appservice_db.py <database file path>`` to convert to
+the new format.
+
Upgrading to v0.7.0
===================
diff --git a/scripts/upgrade_appservice_db.py b/scripts/upgrade_appservice_db.py
new file mode 100644
index 0000000000..ae1b91c64f
--- /dev/null
+++ b/scripts/upgrade_appservice_db.py
@@ -0,0 +1,54 @@
+from synapse.storage import read_schema
+import argparse
+import json
+import sqlite3
+
+
+def do_other_deltas(cursor):
+ cursor.execute("PRAGMA user_version")
+ row = cursor.fetchone()
+
+ if row and row[0]:
+ user_version = row[0]
+ # Run every version since after the current version.
+ for v in range(user_version + 1, 10):
+ print "Running delta: %d" % (v,)
+ sql_script = read_schema("delta/v%d" % (v,))
+ cursor.executescript(sql_script)
+
+
+def update_app_service_table(cur):
+ cur.execute("SELECT id, regex FROM application_services_regex")
+ for row in cur.fetchall():
+ try:
+ print "checking %s..." % row[0]
+ json.loads(row[1])
+ except ValueError:
+ # row isn't in json, make it so.
+ string_regex = row[1]
+ new_regex = json.dumps({
+ "regex": string_regex,
+ "exclusive": True
+ })
+ cur.execute(
+ "UPDATE application_services_regex SET regex=? WHERE id=?",
+ (new_regex, row[0])
+ )
+
+
+def main(dbname):
+ con = sqlite3.connect(dbname)
+ cur = con.cursor()
+ do_other_deltas(cur)
+ update_app_service_table(cur)
+ cur.execute("PRAGMA user_version = 14")
+ cur.close()
+ con.commit()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("database")
+ args = parser.parse_args()
+
+ main(args.database)
diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py
index 381b4cfc4a..a268a6bcc4 100644
--- a/synapse/appservice/__init__.py
+++ b/synapse/appservice/__init__.py
@@ -46,22 +46,34 @@ class ApplicationService(object):
def _check_namespaces(self, namespaces):
# Sanity check that it is of the form:
# {
- # users: ["regex",...],
- # aliases: ["regex",...],
- # rooms: ["regex",...],
+ # users: [ {regex: "[A-z]+.*", exclusive: true}, ...],
+ # aliases: [ {regex: "[A-z]+.*", exclusive: true}, ...],
+ # rooms: [ {regex: "[A-z]+.*", exclusive: true}, ...],
# }
if not namespaces:
return None
for ns in ApplicationService.NS_LIST:
+ if ns not in namespaces:
+ namespaces[ns] = []
+ continue
+
if type(namespaces[ns]) != list:
- raise ValueError("Bad namespace value for '%s'", ns)
- for regex in namespaces[ns]:
- if not isinstance(regex, basestring):
- raise ValueError("Expected string regex for ns '%s'", ns)
+ raise ValueError("Bad namespace value for '%s'" % ns)
+ for regex_obj in namespaces[ns]:
+ if not isinstance(regex_obj, dict):
+ raise ValueError("Expected dict regex for ns '%s'" % ns)
+ if not isinstance(regex_obj.get("exclusive"), bool):
+ raise ValueError(
+ "Expected bool for 'exclusive' in ns '%s'" % ns
+ )
+ if not isinstance(regex_obj.get("regex"), basestring):
+ raise ValueError(
+ "Expected string for 'regex' in ns '%s'" % ns
+ )
return namespaces
- def _matches_regex(self, test_string, namespace_key):
+ def _matches_regex(self, test_string, namespace_key, return_obj=False):
if not isinstance(test_string, basestring):
logger.error(
"Expected a string to test regex against, but got %s",
@@ -69,11 +81,19 @@ class ApplicationService(object):
)
return False
- for regex in self.namespaces[namespace_key]:
- if re.match(regex, test_string):
+ for regex_obj in self.namespaces[namespace_key]:
+ if re.match(regex_obj["regex"], test_string):
+ if return_obj:
+ return regex_obj
return True
return False
+ def _is_exclusive(self, ns_key, test_string):
+ regex_obj = self._matches_regex(test_string, ns_key, return_obj=True)
+ if regex_obj:
+ return regex_obj["exclusive"]
+ return False
+
def _matches_user(self, event, member_list):
if (hasattr(event, "sender") and
self.is_interested_in_user(event.sender)):
@@ -143,5 +163,14 @@ class ApplicationService(object):
def is_interested_in_room(self, room_id):
return self._matches_regex(room_id, ApplicationService.NS_ROOMS)
+ def is_exclusive_user(self, user_id):
+ return self._is_exclusive(ApplicationService.NS_USERS, user_id)
+
+ def is_exclusive_alias(self, alias):
+ return self._is_exclusive(ApplicationService.NS_ALIASES, alias)
+
+ def is_exclusive_room(self, room_id):
+ return self._is_exclusive(ApplicationService.NS_ROOMS, room_id)
+
def __str__(self):
return "ApplicationService: %s" % (self.__dict__,)
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index 11d20a5d2d..f76febee8f 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -232,13 +232,23 @@ class DirectoryHandler(BaseHandler):
@defer.inlineCallbacks
def can_modify_alias(self, alias, user_id=None):
+ # Any application service "interested" in an alias they are regexing on
+ # can modify the alias.
+ # Users can only modify the alias if ALL the interested services have
+ # non-exclusive locks on the alias (or there are no interested services)
services = yield self.store.get_app_services()
interested_services = [
s for s in services if s.is_interested_in_alias(alias.to_string())
]
+
for service in interested_services:
if user_id == service.sender:
- # this user IS the app service
+ # this user IS the app service so they can do whatever they like
defer.returnValue(True)
return
- defer.returnValue(len(interested_services) == 0)
+ elif service.is_exclusive_alias(alias.to_string()):
+ # another service has an exclusive lock on this alias.
+ defer.returnValue(False)
+ return
+ # either no interested services, or no service with an exclusive lock
+ defer.returnValue(True)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 516a936cee..cda4a8502a 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -201,11 +201,12 @@ class RegistrationHandler(BaseHandler):
interested_services = [
s for s in services if s.is_interested_in_user(user_id)
]
- if len(interested_services) > 0:
- raise SynapseError(
- 400, "This user ID is reserved by an application service.",
- errcode=Codes.EXCLUSIVE
- )
+ for service in interested_services:
+ if service.is_exclusive_user(user_id):
+ raise SynapseError(
+ 400, "This user ID is reserved by an application service.",
+ errcode=Codes.EXCLUSIVE
+ )
def _generate_token(self, user_id):
# urlsafe variant uses _ and - so use . as the separator and replace
diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py
index 3bd0c1220c..a4f6159773 100644
--- a/synapse/rest/appservice/v1/register.py
+++ b/synapse/rest/appservice/v1/register.py
@@ -48,18 +48,12 @@ class RegisterRestServlet(AppServiceRestServlet):
400, "Missed required keys: as_token(str) / url(str)."
)
- namespaces = {
- "users": [],
- "rooms": [],
- "aliases": []
- }
-
- if "namespaces" in params:
- self._parse_namespace(namespaces, params["namespaces"], "users")
- self._parse_namespace(namespaces, params["namespaces"], "rooms")
- self._parse_namespace(namespaces, params["namespaces"], "aliases")
-
- app_service = ApplicationService(as_token, as_url, namespaces)
+ try:
+ app_service = ApplicationService(
+ as_token, as_url, params["namespaces"]
+ )
+ except ValueError as e:
+ raise SynapseError(400, e.message)
app_service = yield self.handler.register(app_service)
hs_token = app_service.hs_token
@@ -68,23 +62,6 @@ class RegisterRestServlet(AppServiceRestServlet):
"hs_token": hs_token
}))
- def _parse_namespace(self, target_ns, origin_ns, ns):
- if ns not in target_ns or ns not in origin_ns:
- return # nothing to parse / map through to.
-
- possible_regex_list = origin_ns[ns]
- if not type(possible_regex_list) == list:
- raise SynapseError(400, "Namespace %s isn't an array." % ns)
-
- for regex in possible_regex_list:
- if not isinstance(regex, basestring):
- raise SynapseError(
- 400, "Regex '%s' isn't a string in namespace %s" %
- (regex, ns)
- )
-
- target_ns[ns] = origin_ns[ns]
-
class UnregisterRestServlet(AppServiceRestServlet):
"""Handles AS registration with the home server.
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index d16e7b8fac..3753cd28d0 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -74,7 +74,7 @@ SCHEMAS = [
# Remember to update this number every time an incompatible change is made to
# database schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 13
+SCHEMA_VERSION = 14
dir_path = os.path.abspath(os.path.dirname(__file__))
diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index 97481d113b..e30265750a 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
+import simplejson
+from simplejson import JSONDecodeError
from twisted.internet import defer
from synapse.api.constants import Membership
@@ -25,12 +27,18 @@ from ._base import SQLBaseStore
logger = logging.getLogger(__name__)
+def log_failure(failure):
+ logger.error("Failed to detect application services: %s", failure.value)
+ logger.error(failure.getTraceback())
+
+
class ApplicationServiceStore(SQLBaseStore):
def __init__(self, hs):
super(ApplicationServiceStore, self).__init__(hs)
self.services_cache = []
self.cache_defer = self._populate_cache()
+ self.cache_defer.addErrback(log_failure)
@defer.inlineCallbacks
def unregister_app_service(self, token):
@@ -130,11 +138,11 @@ class ApplicationServiceStore(SQLBaseStore):
)
for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST):
if ns_str in service.namespaces:
- for regex in service.namespaces[ns_str]:
+ for regex_obj in service.namespaces[ns_str]:
txn.execute(
"INSERT INTO application_services_regex("
"as_id, namespace, regex) values(?,?,?)",
- (as_id, ns_int, regex)
+ (as_id, ns_int, simplejson.dumps(regex_obj))
)
return True
@@ -311,10 +319,12 @@ class ApplicationServiceStore(SQLBaseStore):
try:
services[as_token]["namespaces"][
ApplicationService.NS_LIST[ns_int]].append(
- res["regex"]
+ simplejson.loads(res["regex"])
)
except IndexError:
logger.error("Bad namespace enum '%s'. %s", ns_int, res)
+ except JSONDecodeError:
+ logger.error("Bad regex object '%s'", res["regex"])
# TODO get last successful txn id f.e. service
for service in services.values():
diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py
index d12e4f2644..eb7becf725 100644
--- a/tests/appservice/test_appservice.py
+++ b/tests/appservice/test_appservice.py
@@ -18,6 +18,13 @@ from mock import Mock, PropertyMock
from tests import unittest
+def _regex(regex, exclusive=True):
+ return {
+ "regex": regex,
+ "exclusive": exclusive
+ }
+
+
class ApplicationServiceTestCase(unittest.TestCase):
def setUp(self):
@@ -36,21 +43,21 @@ class ApplicationServiceTestCase(unittest.TestCase):
def test_regex_user_id_prefix_match(self):
self.service.namespaces[ApplicationService.NS_USERS].append(
- "@irc_.*"
+ _regex("@irc_.*")
)
self.event.sender = "@irc_foobar:matrix.org"
self.assertTrue(self.service.is_interested(self.event))
def test_regex_user_id_prefix_no_match(self):
self.service.namespaces[ApplicationService.NS_USERS].append(
- "@irc_.*"
+ _regex("@irc_.*")
)
self.event.sender = "@someone_else:matrix.org"
self.assertFalse(self.service.is_interested(self.event))
def test_regex_room_member_is_checked(self):
self.service.namespaces[ApplicationService.NS_USERS].append(
- "@irc_.*"
+ _regex("@irc_.*")
)
self.event.sender = "@someone_else:matrix.org"
self.event.type = "m.room.member"
@@ -59,30 +66,78 @@ class ApplicationServiceTestCase(unittest.TestCase):
def test_regex_room_id_match(self):
self.service.namespaces[ApplicationService.NS_ROOMS].append(
- "!some_prefix.*some_suffix:matrix.org"
+ _regex("!some_prefix.*some_suffix:matrix.org")
)
self.event.room_id = "!some_prefixs0m3th1nGsome_suffix:matrix.org"
self.assertTrue(self.service.is_interested(self.event))
def test_regex_room_id_no_match(self):
self.service.namespaces[ApplicationService.NS_ROOMS].append(
- "!some_prefix.*some_suffix:matrix.org"
+ _regex("!some_prefix.*some_suffix:matrix.org")
)
self.event.room_id = "!XqBunHwQIXUiqCaoxq:matrix.org"
self.assertFalse(self.service.is_interested(self.event))
def test_regex_alias_match(self):
self.service.namespaces[ApplicationService.NS_ALIASES].append(
- "#irc_.*:matrix.org"
+ _regex("#irc_.*:matrix.org")
)
self.assertTrue(self.service.is_interested(
self.event,
aliases_for_event=["#irc_foobar:matrix.org", "#athing:matrix.org"]
))
+ def test_non_exclusive_alias(self):
+ self.service.namespaces[ApplicationService.NS_ALIASES].append(
+ _regex("#irc_.*:matrix.org", exclusive=False)
+ )
+ self.assertFalse(self.service.is_exclusive_alias(
+ "#irc_foobar:matrix.org"
+ ))
+
+ def test_non_exclusive_room(self):
+ self.service.namespaces[ApplicationService.NS_ROOMS].append(
+ _regex("!irc_.*:matrix.org", exclusive=False)
+ )
+ self.assertFalse(self.service.is_exclusive_room(
+ "!irc_foobar:matrix.org"
+ ))
+
+ def test_non_exclusive_user(self):
+ self.service.namespaces[ApplicationService.NS_USERS].append(
+ _regex("@irc_.*:matrix.org", exclusive=False)
+ )
+ self.assertFalse(self.service.is_exclusive_user(
+ "@irc_foobar:matrix.org"
+ ))
+
+ def test_exclusive_alias(self):
+ self.service.namespaces[ApplicationService.NS_ALIASES].append(
+ _regex("#irc_.*:matrix.org", exclusive=True)
+ )
+ self.assertTrue(self.service.is_exclusive_alias(
+ "#irc_foobar:matrix.org"
+ ))
+
+ def test_exclusive_user(self):
+ self.service.namespaces[ApplicationService.NS_USERS].append(
+ _regex("@irc_.*:matrix.org", exclusive=True)
+ )
+ self.assertTrue(self.service.is_exclusive_user(
+ "@irc_foobar:matrix.org"
+ ))
+
+ def test_exclusive_room(self):
+ self.service.namespaces[ApplicationService.NS_ROOMS].append(
+ _regex("!irc_.*:matrix.org", exclusive=True)
+ )
+ self.assertTrue(self.service.is_exclusive_room(
+ "!irc_foobar:matrix.org"
+ ))
+
def test_regex_alias_no_match(self):
self.service.namespaces[ApplicationService.NS_ALIASES].append(
- "#irc_.*:matrix.org"
+ _regex("#irc_.*:matrix.org")
)
self.assertFalse(self.service.is_interested(
self.event,
@@ -91,10 +146,10 @@ class ApplicationServiceTestCase(unittest.TestCase):
def test_regex_multiple_matches(self):
self.service.namespaces[ApplicationService.NS_ALIASES].append(
- "#irc_.*:matrix.org"
+ _regex("#irc_.*:matrix.org")
)
self.service.namespaces[ApplicationService.NS_USERS].append(
- "@irc_.*"
+ _regex("@irc_.*")
)
self.event.sender = "@irc_foobar:matrix.org"
self.assertTrue(self.service.is_interested(
@@ -104,10 +159,10 @@ class ApplicationServiceTestCase(unittest.TestCase):
def test_restrict_to_rooms(self):
self.service.namespaces[ApplicationService.NS_ROOMS].append(
- "!flibble_.*:matrix.org"
+ _regex("!flibble_.*:matrix.org")
)
self.service.namespaces[ApplicationService.NS_USERS].append(
- "@irc_.*"
+ _regex("@irc_.*")
)
self.event.sender = "@irc_foobar:matrix.org"
self.event.room_id = "!wibblewoo:matrix.org"
@@ -118,10 +173,10 @@ class ApplicationServiceTestCase(unittest.TestCase):
def test_restrict_to_aliases(self):
self.service.namespaces[ApplicationService.NS_ALIASES].append(
- "#xmpp_.*:matrix.org"
+ _regex("#xmpp_.*:matrix.org")
)
self.service.namespaces[ApplicationService.NS_USERS].append(
- "@irc_.*"
+ _regex("@irc_.*")
)
self.event.sender = "@irc_foobar:matrix.org"
self.assertFalse(self.service.is_interested(
@@ -132,10 +187,10 @@ class ApplicationServiceTestCase(unittest.TestCase):
def test_restrict_to_senders(self):
self.service.namespaces[ApplicationService.NS_ALIASES].append(
- "#xmpp_.*:matrix.org"
+ _regex("#xmpp_.*:matrix.org")
)
self.service.namespaces[ApplicationService.NS_USERS].append(
- "@irc_.*"
+ _regex("@irc_.*")
)
self.event.sender = "@xmpp_foobar:matrix.org"
self.assertFalse(self.service.is_interested(
@@ -146,7 +201,7 @@ class ApplicationServiceTestCase(unittest.TestCase):
def test_member_list_match(self):
self.service.namespaces[ApplicationService.NS_USERS].append(
- "@irc_.*"
+ _regex("@irc_.*")
)
join_list = [
Mock(
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index fc733d4c79..ca5b92ec85 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -50,9 +50,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
def test_update_and_retrieval_of_service(self):
url = "https://matrix.org/appservices/foobar"
hs_token = "hstok"
- user_regex = ["@foobar_.*:matrix.org"]
- alias_regex = ["#foobar_.*:matrix.org"]
- room_regex = []
+ user_regex = [
+ {"regex": "@foobar_.*:matrix.org", "exclusive": True}
+ ]
+ alias_regex = [
+ {"regex": "#foobar_.*:matrix.org", "exclusive": False}
+ ]
+ room_regex = [
+
+ ]
service = ApplicationService(
url=url, hs_token=hs_token, token=self.as_token, namespaces={
ApplicationService.NS_USERS: user_regex,
|
