diff --git a/synapse/__init__.py b/synapse/__init__.py
index 5a28fe2b82..df0504ac2c 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@ try:
except ImportError:
pass
-__version__ = "0.33.9"
+__version__ = "0.34.0rc1"
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 34382e4e3c..5309899703 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -188,17 +188,33 @@ class Auth(object):
"""
# Can optionally look elsewhere in the request (e.g. headers)
try:
+ ip_addr = self.hs.get_ip_from_request(request)
+ user_agent = request.requestHeaders.getRawHeaders(
+ b"User-Agent",
+ default=[b""]
+ )[0].decode('ascii', 'surrogateescape')
+
+ access_token = self.get_access_token_from_request(
+ request, self.TOKEN_NOT_FOUND_HTTP_STATUS
+ )
+
user_id, app_service = yield self._get_appservice_user_id(request)
if user_id:
request.authenticated_entity = user_id
+
+ if ip_addr and self.hs.config.track_appservice_user_ips:
+ yield self.store.insert_client_ip(
+ user_id=user_id,
+ access_token=access_token,
+ ip=ip_addr,
+ user_agent=user_agent,
+ device_id="dummy-device", # stubbed
+ )
+
defer.returnValue(
synapse.types.create_requester(user_id, app_service=app_service)
)
- access_token = self.get_access_token_from_request(
- request, self.TOKEN_NOT_FOUND_HTTP_STATUS
- )
-
user_info = yield self.get_user_by_access_token(access_token, rights)
user = user_info["user"]
token_id = user_info["token_id"]
@@ -208,11 +224,6 @@ class Auth(object):
# stubbed out.
device_id = user_info.get("device_id")
- ip_addr = self.hs.get_ip_from_request(request)
- user_agent = request.requestHeaders.getRawHeaders(
- b"User-Agent",
- default=[b""]
- )[0].decode('ascii', 'surrogateescape')
if user and access_token and ip_addr:
yield self.store.insert_client_ip(
user_id=user.to_string(),
diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py
index 2430814796..685f15c061 100644
--- a/synapse/appservice/scheduler.py
+++ b/synapse/appservice/scheduler.py
@@ -53,8 +53,8 @@ import logging
from twisted.internet import defer
from synapse.appservice import ApplicationServiceState
+from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.util.logcontext import run_in_background
-from synapse.util.metrics import Measure
logger = logging.getLogger(__name__)
@@ -104,27 +104,34 @@ class _ServiceQueuer(object):
self.clock = clock
def enqueue(self, service, event):
- # if this service isn't being sent something
self.queued_events.setdefault(service.id, []).append(event)
- run_in_background(self._send_request, service)
- @defer.inlineCallbacks
- def _send_request(self, service):
+ # start a sender for this appservice if we don't already have one
+
if service.id in self.requests_in_flight:
return
+ run_as_background_process(
+ "as-sender-%s" % (service.id, ),
+ self._send_request, service,
+ )
+
+ @defer.inlineCallbacks
+ def _send_request(self, service):
+ # sanity-check: we shouldn't get here if this service already has a sender
+ # running.
+ assert(service.id not in self.requests_in_flight)
+
self.requests_in_flight.add(service.id)
try:
while True:
events = self.queued_events.pop(service.id, [])
if not events:
return
-
- with Measure(self.clock, "servicequeuer.send"):
- try:
- yield self.txn_ctrl.send(service, events)
- except Exception:
- logger.exception("AS request failed")
+ try:
+ yield self.txn_ctrl.send(service, events)
+ except Exception:
+ logger.exception("AS request failed")
finally:
self.requests_in_flight.discard(service.id)
@@ -223,7 +230,12 @@ class _Recoverer(object):
self.backoff_counter = 1
def recover(self):
- self.clock.call_later((2 ** self.backoff_counter), self.retry)
+ def _retry():
+ run_as_background_process(
+ "as-recoverer-%s" % (self.service.id,),
+ self.retry,
+ )
+ self.clock.call_later((2 ** self.backoff_counter), _retry)
def _backoff(self):
# cap the backoff to be around 8.5min => (2^9) = 512 secs
diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index 3b161d708a..c21cb3dd87 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -33,11 +33,16 @@ class AppServiceConfig(Config):
def read_config(self, config):
self.app_service_config_files = config.get("app_service_config_files", [])
self.notify_appservices = config.get("notify_appservices", True)
+ self.track_appservice_user_ips = config.get("track_appservice_user_ips", False)
def default_config(cls, **kwargs):
return """\
# A list of application service config file to use
app_service_config_files: []
+
+ # Whether or not to track application service IP addresses. Implicitly
+ # enables MAU tracking for application service users.
+ track_appservice_user_ips: False
"""
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 5ff9ac288d..4a5b902f8e 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -62,6 +62,11 @@ class ServerConfig(Config):
# master, potentially causing inconsistency.
self.enable_media_repo = config.get("enable_media_repo", True)
+ # whether to enable search. If disabled, new entries will not be inserted
+ # into the search tables and they will not be indexed. Users will receive
+ # errors when attempting to search for messages.
+ self.enable_search = config.get("enable_search", True)
+
self.filter_timeline_limit = config.get("filter_timeline_limit", -1)
# Whether we should block invites sent to users on this server
@@ -384,7 +389,12 @@ class ServerConfig(Config):
# mau_limit_reserved_threepids:
# - medium: 'email'
# address: 'reserved_user@example.com'
-
+ #
+ # Room searching
+ #
+ # If disabled, new messages will not be indexed for searching and users
+ # will receive errors when searching for messages. Defaults to enabled.
+ # enable_search: true
""" % locals()
def read_arguments(self, args):
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index d2beb275cf..015909bb26 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -217,7 +217,19 @@ class RegistrationHandler(BaseHandler):
user_id = None
token = None
attempts += 1
+ if not self.hs.config.user_consent_at_registration:
+ yield self._auto_join_rooms(user_id)
+ defer.returnValue((user_id, token))
+
+ @defer.inlineCallbacks
+ def _auto_join_rooms(self, user_id):
+ """Automatically joins users to auto join rooms - creating the room in the first place
+ if the user is the first to be created.
+
+ Args:
+ user_id(str): The user to join
+ """
# auto-join the user to any rooms we're supposed to dump them into
fake_requester = create_requester(user_id)
@@ -226,7 +238,6 @@ class RegistrationHandler(BaseHandler):
if self.hs.config.autocreate_auto_join_rooms:
count = yield self.store.count_all_users()
should_auto_create_rooms = count == 1
-
for r in self.hs.config.auto_join_rooms:
try:
if should_auto_create_rooms:
@@ -256,7 +267,15 @@ class RegistrationHandler(BaseHandler):
except Exception as e:
logger.error("Failed to join new user to %r: %r", r, e)
- defer.returnValue((user_id, token))
+ @defer.inlineCallbacks
+ def post_consent_actions(self, user_id):
+ """A series of registration actions that can only be carried out once consent
+ has been granted
+
+ Args:
+ user_id (str): The user to join
+ """
+ yield self._auto_join_rooms(user_id)
@defer.inlineCallbacks
def appservice_register(self, user_localpart, as_token):
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index 80e7b15de8..ec936bbb4e 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -50,6 +50,9 @@ class SearchHandler(BaseHandler):
dict to be returned to the client with results of search
"""
+ if not self.hs.config.enable_search:
+ raise SynapseError(400, "Search is disabled on this homeserver")
+
batch_group = None
batch_group_key = None
batch_token = None
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index 0010699d31..f6b4a85e40 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -27,7 +27,7 @@ from twisted.web.client import PartialDownloadError
from synapse.api.errors import Codes, LoginError, SynapseError
from synapse.http.server import finish_request
-from synapse.http.servlet import parse_json_object_from_request
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.types import UserID
from synapse.util.msisdn import phone_number_to_msisdn
@@ -83,6 +83,7 @@ class LoginRestServlet(ClientV1RestServlet):
PATTERNS = client_path_patterns("/login$")
SAML2_TYPE = "m.login.saml2"
CAS_TYPE = "m.login.cas"
+ SSO_TYPE = "m.login.sso"
TOKEN_TYPE = "m.login.token"
JWT_TYPE = "m.login.jwt"
@@ -105,6 +106,10 @@ class LoginRestServlet(ClientV1RestServlet):
if self.saml2_enabled:
flows.append({"type": LoginRestServlet.SAML2_TYPE})
if self.cas_enabled:
+ flows.append({"type": LoginRestServlet.SSO_TYPE})
+
+ # we advertise CAS for backwards compat, though MSC1721 renamed it
+ # to SSO.
flows.append({"type": LoginRestServlet.CAS_TYPE})
# While its valid for us to advertise this login type generally,
@@ -384,11 +389,11 @@ class SAML2RestServlet(ClientV1RestServlet):
defer.returnValue((200, {"status": "not_authenticated"}))
-class CasRedirectServlet(ClientV1RestServlet):
- PATTERNS = client_path_patterns("/login/cas/redirect", releases=())
+class CasRedirectServlet(RestServlet):
+ PATTERNS = client_path_patterns("/login/(cas|sso)/redirect")
def __init__(self, hs):
- super(CasRedirectServlet, self).__init__(hs)
+ super(CasRedirectServlet, self).__init__()
self.cas_server_url = hs.config.cas_server_url.encode('ascii')
self.cas_service_url = hs.config.cas_service_url.encode('ascii')
diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index 9382b1f124..c654f9b5f0 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -42,7 +42,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
@defer.inlineCallbacks
def on_PUT(self, request):
- spec = _rule_spec_from_path(request.postpath)
+ spec = _rule_spec_from_path([x.decode('utf8') for x in request.postpath])
try:
priority_class = _priority_class_from_spec(spec)
except InvalidRuleException as e:
@@ -103,7 +103,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
@defer.inlineCallbacks
def on_DELETE(self, request):
- spec = _rule_spec_from_path(request.postpath)
+ spec = _rule_spec_from_path([x.decode('utf8') for x in request.postpath])
requester = yield self.auth.get_user_by_req(request)
user_id = requester.user.to_string()
@@ -134,7 +134,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
rules = format_push_rules_for_user(requester.user, rules)
- path = request.postpath[1:]
+ path = [x.decode('utf8') for x in request.postpath][1:]
if path == []:
# we're a reference impl: pedantry is our job.
@@ -142,11 +142,10 @@ class PushRuleRestServlet(ClientV1RestServlet):
PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR
)
- if path[0] == b'':
+ if path[0] == '':
defer.returnValue((200, rules))
- elif path[0] == b'global':
- path = [x.decode('ascii') for x in path[1:]]
- result = _filter_ruleset_with_path(rules['global'], path)
+ elif path[0] == 'global':
+ result = _filter_ruleset_with_path(rules['global'], path[1:])
defer.returnValue((200, result))
else:
raise UnrecognizedRequestError()
@@ -190,12 +189,24 @@ class PushRuleRestServlet(ClientV1RestServlet):
def _rule_spec_from_path(path):
+ """Turn a sequence of path components into a rule spec
+
+ Args:
+ path (sequence[unicode]): the URL path components.
+
+ Returns:
+ dict: rule spec dict, containing scope/template/rule_id entries,
+ and possibly attr.
+
+ Raises:
+ UnrecognizedRequestError if the path components cannot be parsed.
+ """
if len(path) < 2:
raise UnrecognizedRequestError()
- if path[0] != b'pushrules':
+ if path[0] != 'pushrules':
raise UnrecognizedRequestError()
- scope = path[1].decode('ascii')
+ scope = path[1]
path = path[2:]
if scope != 'global':
raise UnrecognizedRequestError()
@@ -203,13 +214,13 @@ def _rule_spec_from_path(path):
if len(path) == 0:
raise UnrecognizedRequestError()
- template = path[0].decode('ascii')
+ template = path[0]
path = path[1:]
if len(path) == 0 or len(path[0]) == 0:
raise UnrecognizedRequestError()
- rule_id = path[0].decode('ascii')
+ rule_id = path[0]
spec = {
'scope': scope,
@@ -220,7 +231,7 @@ def _rule_spec_from_path(path):
path = path[1:]
if len(path) > 0 and len(path[0]) > 0:
- spec['attr'] = path[0].decode('ascii')
+ spec['attr'] = path[0]
return spec
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index b84f0260f2..4c07ae7f45 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -142,7 +142,7 @@ class PushersRemoveRestServlet(RestServlet):
To allow pusher to be delete by clicking a link (ie. GET request)
"""
PATTERNS = client_path_patterns("/pushers/remove$")
- SUCCESS_HTML = "<html><body>You have been unsubscribed</body><html>"
+ SUCCESS_HTML = b"<html><body>You have been unsubscribed</body><html>"
def __init__(self, hs):
super(PushersRemoveRestServlet, self).__init__()
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 0515715f7c..aec0c6b075 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -457,6 +457,7 @@ class RegisterRestServlet(RestServlet):
yield self.store.user_set_consent_version(
registered_user_id, self.hs.config.user_consent_version,
)
+ yield self.registration_handler.post_consent_actions(registered_user_id)
defer.returnValue((200, return_dict))
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index ad525b22e1..80611cfe84 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -89,6 +89,7 @@ class ConsentResource(Resource):
self.hs = hs
self.store = hs.get_datastore()
+ self.registration_handler = hs.get_handlers().registration_handler
# this is required by the request_handler wrapper
self.clock = hs.get_clock()
@@ -199,6 +200,7 @@ class ConsentResource(Resource):
if e.code != 404:
raise
raise NotFoundError("Unknown user")
+ yield self.registration_handler.post_consent_actions(qualified_user_id)
try:
self._render_template(request, "success.html")
diff --git a/synapse/state/v1.py b/synapse/state/v1.py
index 70a981f4a2..19e091ce3b 100644
--- a/synapse/state/v1.py
+++ b/synapse/state/v1.py
@@ -298,6 +298,8 @@ def _resolve_normal_events(events, auth_events):
def _ordered_events(events):
def key_func(e):
- return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest()
+ # we have to use utf-8 rather than ascii here because it turns out we allow
+ # people to send us events with non-ascii event IDs :/
+ return -int(e.depth), hashlib.sha1(e.event_id.encode('utf-8')).hexdigest()
return sorted(events, key=key_func)
diff --git a/synapse/static/client/login/index.html b/synapse/static/client/login/index.html
index 96c8723cab..bcb6bc6bb7 100644
--- a/synapse/static/client/login/index.html
+++ b/synapse/static/client/login/index.html
@@ -12,35 +12,30 @@
<h1>Log in with one of the following methods</h1>
<span id="feedback" style="color: #f00"></span>
- <br/>
- <br/>
<div id="loading">
<img src="spinner.gif" />
</div>
- <div id="cas_flow" class="login_flow" style="display:none"
- onclick="gotoCas(); return false;">
- CAS Authentication: <button id="cas_button" style="margin: 10px">Log in</button>
+ <div id="sso_flow" class="login_flow" style="display:none">
+ Single-sign on:
+ <form id="sso_form" action="/_matrix/client/r0/login/sso/redirect" method="get">
+ <input id="sso_redirect_url" type="hidden" name="redirectUrl" value=""/>
+ <input type="submit" value="Log in"/>
+ </form>
</div>
- <br/>
-
- <form id="password_form" class="login_flow" style="display:none"
- onsubmit="matrixLogin.password_login(); return false;">
- <div>
- Password Authentication:<br/>
-
- <div style="text-align: center">
- <input id="user_id" size="32" type="text" placeholder="Matrix ID (e.g. bob)" autocapitalize="off" autocorrect="off" />
- <br/>
- <input id="password" size="32" type="password" placeholder="Password"/>
- <br/>
+ <div id="password_flow" class="login_flow" style="display:none">
+ Password Authentication:
+ <form onsubmit="matrixLogin.password_login(); return false;">
+ <input id="user_id" size="32" type="text" placeholder="Matrix ID (e.g. bob)" autocapitalize="off" autocorrect="off" />
+ <br/>
+ <input id="password" size="32" type="password" placeholder="Password"/>
+ <br/>
- <button type="submit" style="margin: 10px">Log in</button>
- </div>
- </div>
- </form>
+ <input type="submit" value="Log in"/>
+ </form>
+ </div>
<div id="no_login_types" type="button" class="login_flow" style="display:none">
Log in currently unavailable.
diff --git a/synapse/static/client/login/js/login.js b/synapse/static/client/login/js/login.js
index bfb7386035..3a958749a1 100644
--- a/synapse/static/client/login/js/login.js
+++ b/synapse/static/client/login/js/login.js
@@ -1,7 +1,8 @@
window.matrixLogin = {
- endpoint: location.origin + "/_matrix/client/api/v1/login",
+ endpoint: location.origin + "/_matrix/client/r0/login",
serverAcceptsPassword: false,
- serverAcceptsCas: false
+ serverAcceptsCas: false,
+ serverAcceptsSso: false,
};
var submitPassword = function(user, pwd) {
@@ -40,12 +41,6 @@ var errorFunc = function(err) {
}
};
-var gotoCas = function() {
- var this_page = window.location.origin + window.location.pathname;
- var redirect_url = matrixLogin.endpoint + "/cas/redirect?redirectUrl=" + encodeURIComponent(this_page);
- window.location.replace(redirect_url);
-}
-
var setFeedbackString = function(text) {
$("#feedback").text(text);
};
@@ -53,12 +48,18 @@ var setFeedbackString = function(text) {
var show_login = function() {
$("#loading").hide();
+ var this_page = window.location.origin + window.location.pathname;
+ $("#sso_redirect_url").val(encodeURIComponent(this_page));
+
if (matrixLogin.serverAcceptsPassword) {
- $("#password_form").show();
+ $("#password_flow").show();
}
- if (matrixLogin.serverAcceptsCas) {
- $("#cas_flow").show();
+ if (matrixLogin.serverAcceptsSso) {
+ $("#sso_flow").show();
+ } else if (matrixLogin.serverAcceptsCas) {
+ $("#sso_form").attr("action", "/_matrix/client/r0/login/cas/redirect");
+ $("#sso_flow").show();
}
if (!matrixLogin.serverAcceptsPassword && !matrixLogin.serverAcceptsCas) {
@@ -67,8 +68,8 @@ var show_login = function() {
};
var show_spinner = function() {
- $("#password_form").hide();
- $("#cas_flow").hide();
+ $("#password_flow").hide();
+ $("#sso_flow").hide();
$("#no_login_types").hide();
$("#loading").show();
};
@@ -84,7 +85,10 @@ var fetch_info = function(cb) {
matrixLogin.serverAcceptsCas = true;
console.log("Server accepts CAS");
}
-
+ if ("m.login.sso" === flow.type) {
+ matrixLogin.serverAcceptsSso = true;
+ console.log("Server accepts SSO");
+ }
if ("m.login.password" === flow.type) {
matrixLogin.serverAcceptsPassword = true;
console.log("Server accepts password");
diff --git a/synapse/static/client/login/style.css b/synapse/static/client/login/style.css
index 73da0b5117..1cce5ed950 100644
--- a/synapse/static/client/login/style.css
+++ b/synapse/static/client/login/style.css
@@ -19,30 +19,23 @@ a:hover { color: #000; }
a:active { color: #000; }
input {
- width: 90%
-}
-
-textarea, input {
- font-family: inherit;
- font-size: inherit;
margin: 5px;
}
-.smallPrint {
- color: #888;
- font-size: 9pt ! important;
- font-style: italic ! important;
+textbox, input[type="text"], input[type="password"] {
+ width: 90%;
}
-.g-recaptcha div {
- margin: auto;
+form {
+ text-align: center;
+ margin: 10px 0 0 0;
}
.login_flow {
+ width: 300px;
text-align: left;
padding: 10px;
margin-bottom: 40px;
- display: inline-block;
-webkit-border-radius: 10px;
-moz-border-radius: 10px;
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 53c685c173..b23fb7e56c 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -119,7 +119,6 @@ class DataStore(RoomMemberStore, RoomStore,
db_conn, "device_lists_stream", "stream_id",
)
- self._transaction_id_gen = IdGenerator(db_conn, "sent_transactions", "id")
self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id")
self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id")
self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id")
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index d9d0255d0b..1d3069b143 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -29,6 +29,7 @@ from synapse.api.errors import StoreError
from synapse.storage.engines import PostgresEngine
from synapse.util.caches.descriptors import Cache
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
+from synapse.util.stringutils import exception_to_unicode
logger = logging.getLogger(__name__)
@@ -249,32 +250,32 @@ class SQLBaseStore(object):
except self.database_engine.module.OperationalError as e:
# This can happen if the database disappears mid
# transaction.
- logger.warn(
+ logger.warning(
"[TXN OPERROR] {%s} %s %d/%d",
- name, e, i, N
+ name, exception_to_unicode(e), i, N
)
if i < N:
i += 1
try:
conn.rollback()
except self.database_engine.module.Error as e1:
- logger.warn(
+ logger.warning(
"[TXN EROLL] {%s} %s",
- name, e1,
+ name, exception_to_unicode(e1),
)
continue
raise
except self.database_engine.module.DatabaseError as e:
if self.database_engine.is_deadlock(e):
- logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N)
+ logger.warning("[TXN DEADLOCK] {%s} %d/%d", name, i, N)
if i < N:
i += 1
try:
conn.rollback()
except self.database_engine.module.Error as e1:
- logger.warn(
+ logger.warning(
"[TXN EROLL] {%s} %s",
- name, e1,
+ name, exception_to_unicode(e1),
)
continue
raise
@@ -849,9 +850,9 @@ class SQLBaseStore(object):
rowcount = cls._simple_update_txn(txn, table, keyvalues, updatevalues)
if rowcount == 0:
- raise StoreError(404, "No row found")
+ raise StoreError(404, "No row found (%s)" % (table,))
if rowcount > 1:
- raise StoreError(500, "More than one row matched")
+ raise StoreError(500, "More than one row matched (%s)" % (table,))
@staticmethod
def _simple_select_one_txn(txn, table, keyvalues, retcols,
@@ -868,9 +869,9 @@ class SQLBaseStore(object):
if not row:
if allow_none:
return None
- raise StoreError(404, "No row found")
+ raise StoreError(404, "No row found (%s)" % (table,))
if txn.rowcount > 1:
- raise StoreError(500, "More than one row matched")
+ raise StoreError(500, "More than one row matched (%s)" % (table,))
return dict(zip(retcols, row))
@@ -902,9 +903,9 @@ class SQLBaseStore(object):
txn.execute(sql, list(keyvalues.values()))
if txn.rowcount == 0:
- raise StoreError(404, "No row found")
+ raise StoreError(404, "No row found (%s)" % (table,))
if txn.rowcount > 1:
- raise StoreError(500, "more than one row matched")
+ raise StoreError(500, "More than one row matched (%s)" % (table,))
def _simple_delete(self, table, keyvalues, desc):
return self.runInteraction(
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index c353b11c9a..479e01ddc1 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -34,8 +34,9 @@ class MonthlyActiveUsersStore(SQLBaseStore):
self.hs = hs
self.reserved_users = ()
# Do not add more reserved users than the total allowable number
- self._initialise_reserved_users(
- dbconn.cursor(),
+ self._new_transaction(
+ dbconn, "initialise_mau_threepids", [], [],
+ self._initialise_reserved_users,
hs.config.mau_limits_reserved_threepids[:self.hs.config.max_mau_value],
)
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index bd740e1e45..fa36daac52 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
# Remember to update this number every time a change is made to database
# schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 52
+SCHEMA_VERSION = 53
dir_path = os.path.abspath(os.path.dirname(__file__))
@@ -257,7 +257,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files,
module.run_create(cur, database_engine)
if not is_empty:
module.run_upgrade(cur, database_engine, config=config)
- elif ext == ".pyc":
+ elif ext == ".pyc" or file_name == "__pycache__":
# Sometimes .pyc files turn up anyway even though we've
# disabled their generation; e.g. from distribution package
# installers. Silently skip it
diff --git a/synapse/storage/schema/delta/34/sent_txn_purge.py b/synapse/storage/schema/delta/34/sent_txn_purge.py
deleted file mode 100644
index 0ffab10b6f..0000000000
--- a/synapse/storage/schema/delta/34/sent_txn_purge.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from synapse.storage.engines import PostgresEngine
-
-logger = logging.getLogger(__name__)
-
-
-def run_create(cur, database_engine, *args, **kwargs):
- if isinstance(database_engine, PostgresEngine):
- cur.execute("TRUNCATE sent_transactions")
- else:
- cur.execute("DELETE FROM sent_transactions")
-
- cur.execute("CREATE INDEX sent_transactions_ts ON sent_transactions(ts)")
-
-
-def run_upgrade(cur, database_engine, *args, **kwargs):
- pass
diff --git a/synapse/storage/schema/delta/11/v11.sql b/synapse/storage/schema/delta/53/drop_sent_transactions.sql
index e7b4f90127..e372f5a44a 100644
--- a/synapse/storage/schema/delta/11/v11.sql
+++ b/synapse/storage/schema/delta/53/drop_sent_transactions.sql
@@ -1,4 +1,4 @@
-/* Copyright 2015, 2016 OpenMarket Ltd
+/* Copyright 2018 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,4 +13,4 @@
* limitations under the License.
*/
-CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
\ No newline at end of file
+DROP TABLE IF EXISTS sent_transactions;
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index a3f4a0a790..f6a058832e 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -25,25 +25,6 @@ CREATE TABLE IF NOT EXISTS received_transactions(
CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
-
--- Stores what transactions we've sent, what their response was (if we got one) and whether we have
--- since referenced the transaction in another outgoing transaction
-CREATE TABLE IF NOT EXISTS sent_transactions(
- id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
- transaction_id TEXT,
- destination TEXT,
- response_code INTEGER DEFAULT 0,
- response_json TEXT,
- ts BIGINT
-);
-
-CREATE INDEX sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id);
--- So that we can do an efficient look up of all transactions that have yet to be successfully
--- sent.
-CREATE INDEX sent_transaction_sent ON sent_transactions(response_code);
-
-
-- For sent transactions only.
CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
transaction_id INTEGER,
diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql
index 14b67cce25..17e67bedac 100644
--- a/synapse/storage/schema/full_schemas/16/transactions.sql
+++ b/synapse/storage/schema/full_schemas/16/transactions.sql
@@ -25,25 +25,6 @@ CREATE TABLE IF NOT EXISTS received_transactions(
CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
-
--- Stores what transactions we've sent, what their response was (if we got one) and whether we have
--- since referenced the transaction in another outgoing transaction
-CREATE TABLE IF NOT EXISTS sent_transactions(
- id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
- transaction_id TEXT,
- destination TEXT,
- response_code INTEGER DEFAULT 0,
- response_json TEXT,
- ts BIGINT
-);
-
-CREATE INDEX sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id);
--- So that we can do an efficient look up of all transactions that have yet to be successfully
--- sent.
-CREATE INDEX sent_transaction_sent ON sent_transactions(response_code);
-
-
-- For sent transactions only.
CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
transaction_id INTEGER,
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index a35291a3f6..ad01071318 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -45,6 +45,10 @@ class SearchStore(BackgroundUpdateStore):
def __init__(self, db_conn, hs):
super(SearchStore, self).__init__(db_conn, hs)
+
+ if not hs.config.enable_search:
+ return
+
self.register_background_update_handler(
self.EVENT_SEARCH_UPDATE_NAME, self._background_reindex_search
)
@@ -316,6 +320,8 @@ class SearchStore(BackgroundUpdateStore):
entries (iterable[SearchEntry]):
entries to be added to the table
"""
+ if not self.hs.config.enable_search:
+ return
if isinstance(self.database_engine, PostgresEngine):
sql = (
"INSERT INTO event_search"
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index 6f318c6a29..fdcb375f95 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -16,7 +16,8 @@
import random
import string
-from six import PY3
+import six
+from six import PY2, PY3
from six.moves import range
_string_with_symbols = (
@@ -71,3 +72,39 @@ def to_ascii(s):
return s.encode("ascii")
except UnicodeEncodeError:
return s
+
+
+def exception_to_unicode(e):
+ """Helper function to extract the text of an exception as a unicode string
+
+ Args:
+ e (Exception): exception to be stringified
+
+ Returns:
+ unicode
+ """
+ # urgh, this is a mess. The basic problem here is that psycopg2 constructs its
+ # exceptions with PyErr_SetString, with a (possibly non-ascii) argument. str() will
+ # then produce the raw byte sequence. Under Python 2, this will then cause another
+ # error if it gets mixed with a `unicode` object, as per
+ # https://github.com/matrix-org/synapse/issues/4252
+
+ # First of all, if we're under python3, everything is fine because it will sort this
+ # nonsense out for us.
+ if not PY2:
+ return str(e)
+
+ # otherwise let's have a stab at decoding the exception message. We'll circumvent
+ # Exception.__str__(), which would explode if someone raised Exception(u'non-ascii')
+ # and instead look at what is in the args member.
+
+ if len(e.args) == 0:
+ return u""
+ elif len(e.args) > 1:
+ return six.text_type(repr(e.args))
+
+ msg = e.args[0]
+ if isinstance(msg, bytes):
+ return msg.decode('utf-8', errors='replace')
+ else:
+ return msg
|
