diff --git a/.gitignore b/.gitignore
index 4c336b710d..f8c4000134 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,3 +43,6 @@ build/
localhost-800*/
static/client/register/register_config.js
.tox
+
+env/
+*.config
diff --git a/CHANGES.rst b/CHANGES.rst
index e6d1a37307..2ec10516fd 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,4 +1,54 @@
-Changes in synapse v0.10.0-rc1 (2015-08-20)
+Changes in synapse v0.10.0 (2015-09-03)
+=======================================
+
+No change from release candidate.
+
+Changes in synapse v0.10.0-rc6 (2015-09-02)
+===========================================
+
+* Remove some of the old database upgrade scripts.
+* Fix database port script to work with newly created sqlite databases.
+
+Changes in synapse v0.10.0-rc5 (2015-08-27)
+===========================================
+
+* Fix bug that broke downloading files with ascii filenames across federation.
+
+Changes in synapse v0.10.0-rc4 (2015-08-27)
+===========================================
+
+* Allow UTF-8 filenames for upload. (PR #259)
+
+Changes in synapse v0.10.0-rc3 (2015-08-25)
+===========================================
+
+* Add ``--keys-directory`` config option to specify where files such as
+ certs and signing keys should be stored in, when using ``--generate-config``
+ or ``--generate-keys``. (PR #250)
+* Allow ``--config-path`` to specify a directory, causing synapse to use all
+ \*.yaml files in the directory as config files. (PR #249)
+* Add ``web_client_location`` config option to specify static files to be
+ hosted by synapse under ``/_matrix/client``. (PR #245)
+* Add helper utility to synapse to read and parse the config files and extract
+ the value of a given key. For example::
+
+ $ python -m synapse.config read server_name -c homeserver.yaml
+ localhost
+
+ (PR #246)
+
+
+Changes in synapse v0.10.0-rc2 (2015-08-24)
+===========================================
+
+* Fix bug where we incorrectly populated the ``event_forward_extremities``
+ table, resulting in problems joining large remote rooms (e.g.
+ ``#matrix:matrix.org``)
+* Reduce the number of times we wake up pushers by not listening for presence
+ or typing events, reducing the CPU cost of each pusher.
+
+
+Changes in synapse v0.10.0-rc1 (2015-08-21)
===========================================
Also see v0.9.4-rc1 changelog, which has been amalgamated into this release.
@@ -10,6 +60,9 @@ General:
(PR #208)
* Add support for logging in with email address (PR #234)
* Add support for new ``m.room.canonical_alias`` event. (PR #233)
+* Change synapse to treat user IDs case insensitively during registration and
+ login. (If two users already exist with case insensitive matching user ids,
+ synapse will continue to require them to specify their user ids exactly.)
* Error if a user tries to register with an email already in use. (PR #211)
* Add extra and improve existing caches (PR #212, #219, #226, #228)
* Batch various storage request (PR #226, #228)
diff --git a/README.rst b/README.rst
index d8d179135b..6c8431aa86 100644
--- a/README.rst
+++ b/README.rst
@@ -94,6 +94,7 @@ Synapse is the reference python/twisted Matrix homeserver implementation.
System requirements:
- POSIX-compliant system (tested on Linux & OS X)
- Python 2.7
+- At least 512 MB RAM.
Synapse is written in python but some of the libraries is uses are written in
C. So before we can install synapse itself we need a working C compiler and the
@@ -120,6 +121,7 @@ To install the synapse homeserver run::
virtualenv -p python2.7 ~/.synapse
source ~/.synapse/bin/activate
+ pip install --upgrade setuptools
pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master
This installs synapse, along with the libraries it uses, into a virtual
@@ -284,6 +286,11 @@ may need to manually upgrade it::
sudo pip install --upgrade pip
+Installing may fail with ``mock requires setuptools>=17.1. Aborting installation``.
+You can fix this by upgrading setuptools::
+
+ pip install --upgrade setuptools
+
If pip crashes mid-installation for reason (e.g. lost terminal), pip may
refuse to run until you remove the temporary installation directory it
created. To reset the installation::
diff --git a/docs/postgres.rst b/docs/postgres.rst
index 19d8391115..b5027fefb0 100644
--- a/docs/postgres.rst
+++ b/docs/postgres.rst
@@ -55,9 +55,8 @@ Porting from SQLite
Overview
~~~~~~~~
-The script ``port_from_sqlite_to_postgres.py`` allows porting an existing
-synapse server backed by SQLite to using PostgreSQL. This is done in as a two
-phase process:
+The script ``synapse_port_db`` allows porting an existing synapse server
+backed by SQLite to using PostgreSQL. This is done in as a two phase process:
1. Copy the existing SQLite database to a separate location (while the server
is down) and running the port script against that offline database.
@@ -86,8 +85,7 @@ Assuming your new config file (as described in the section *Synapse config*)
is named ``homeserver-postgres.yaml`` and the SQLite snapshot is at
``homeserver.db.snapshot`` then simply run::
- python scripts/port_from_sqlite_to_postgres.py \
- --sqlite-database homeserver.db.snapshot \
+ synapse_port_db --sqlite-database homeserver.db.snapshot \
--postgres-config homeserver-postgres.yaml
The flag ``--curses`` displays a coloured curses progress UI.
@@ -100,8 +98,7 @@ To complete the conversion shut down the synapse server and run the port
script one last time, e.g. if the SQLite database is at ``homeserver.db``
run::
- python scripts/port_from_sqlite_to_postgres.py \
- --sqlite-database homeserver.db \
+ synapse_port_db --sqlite-database homeserver.db \
--postgres-config database_config.yaml
Once that has completed, change the synapse config to point at the PostgreSQL
diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py
index b889ac7fa7..4fa8792a5f 100644
--- a/scripts-dev/check_auth.py
+++ b/scripts-dev/check_auth.py
@@ -56,10 +56,9 @@ if __name__ == '__main__':
js = json.load(args.json)
-
auth = Auth(Mock())
check_auth(
auth,
[FrozenEvent(d) for d in js["auth_chain"]],
- [FrozenEvent(d) for d in js["pdus"]],
+ [FrozenEvent(d) for d in js.get("pdus", [])],
)
diff --git a/scripts/database-prepare-for-0.0.1.sh b/scripts/database-prepare-for-0.0.1.sh
deleted file mode 100755
index 43d759a5cd..0000000000
--- a/scripts/database-prepare-for-0.0.1.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-# This is will prepare a synapse database for running with v0.0.1 of synapse.
-# It will store all the user information, but will *delete* all messages and
-# room data.
-
-set -e
-
-cp "$1" "$1.bak"
-
-DUMP=$(sqlite3 "$1" << 'EOF'
-.dump users
-.dump access_tokens
-.dump presence
-.dump profiles
-EOF
-)
-
-rm "$1"
-
-sqlite3 "$1" <<< "$DUMP"
diff --git a/scripts/database-prepare-for-0.5.0.sh b/scripts/database-prepare-for-0.5.0.sh
deleted file mode 100755
index e824cb583e..0000000000
--- a/scripts/database-prepare-for-0.5.0.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-# This is will prepare a synapse database for running with v0.5.0 of synapse.
-# It will store all the user information, but will *delete* all messages and
-# room data.
-
-set -e
-
-cp "$1" "$1.bak"
-
-DUMP=$(sqlite3 "$1" << 'EOF'
-.dump users
-.dump access_tokens
-.dump presence
-.dump profiles
-EOF
-)
-
-rm "$1"
-
-sqlite3 "$1" <<< "$DUMP"
diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/synapse_port_db
index e7ed4c309b..6aba72e459 100755
--- a/scripts/port_from_sqlite_to_postgres.py
+++ b/scripts/synapse_port_db
@@ -29,7 +29,7 @@ import traceback
import yaml
-logger = logging.getLogger("port_from_sqlite_to_postgres")
+logger = logging.getLogger("synapse_port_db")
BOOLEAN_COLUMNS = {
@@ -412,14 +412,17 @@ class Porter(object):
self._convert_rows("sent_transactions", headers, rows)
inserted_rows = len(rows)
- max_inserted_rowid = max(r[0] for r in rows)
+ if inserted_rows:
+ max_inserted_rowid = max(r[0] for r in rows)
- def insert(txn):
- self.postgres_store.insert_many_txn(
- txn, "sent_transactions", headers[1:], rows
- )
+ def insert(txn):
+ self.postgres_store.insert_many_txn(
+ txn, "sent_transactions", headers[1:], rows
+ )
- yield self.postgres_store.execute(insert)
+ yield self.postgres_store.execute(insert)
+ else:
+ max_inserted_rowid = 0
def get_start_id(txn):
txn.execute(
diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py
deleted file mode 100755
index cd4be28b86..0000000000
--- a/scripts/upgrade_db_to_v0.6.0.py
+++ /dev/null
@@ -1,326 +0,0 @@
-#!/usr/bin/env python
-from synapse.storage import SCHEMA_VERSION, read_schema
-from synapse.storage._base import SQLBaseStore
-from synapse.storage.signatures import SignatureStore
-from synapse.storage.event_federation import EventFederationStore
-
-from unpaddedbase64 import encode_base64, decode_base64
-
-from synapse.crypto.event_signing import compute_event_signature
-
-from synapse.events.builder import EventBuilder
-from synapse.events.utils import prune_event
-
-from synapse.crypto.event_signing import check_event_content_hash
-
-from signedjson.sign import verify_signed_json, SignatureVerifyException
-from signedjson.key import decode_verify_key_bytes
-
-from canonicaljson import encode_canonical_json
-
-import argparse
-# import dns.resolver
-import hashlib
-import httplib
-import json
-import sqlite3
-import urllib2
-
-
-delta_sql = """
-CREATE TABLE IF NOT EXISTS event_json(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- internal_metadata NOT NULL,
- json BLOB NOT NULL,
- CONSTRAINT ev_j_uniq UNIQUE (event_id)
-);
-
-CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id);
-CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
-
-PRAGMA user_version = 10;
-"""
-
-
-class Store(object):
- _get_event_signatures_txn = SignatureStore.__dict__["_get_event_signatures_txn"]
- _get_event_content_hashes_txn = SignatureStore.__dict__["_get_event_content_hashes_txn"]
- _get_event_reference_hashes_txn = SignatureStore.__dict__["_get_event_reference_hashes_txn"]
- _get_prev_event_hashes_txn = SignatureStore.__dict__["_get_prev_event_hashes_txn"]
- _get_prev_events_and_state = EventFederationStore.__dict__["_get_prev_events_and_state"]
- _get_auth_events = EventFederationStore.__dict__["_get_auth_events"]
- cursor_to_dict = SQLBaseStore.__dict__["cursor_to_dict"]
- _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"]
- _simple_select_list_txn = SQLBaseStore.__dict__["_simple_select_list_txn"]
- _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"]
-
- def _generate_event_json(self, txn, rows):
- events = []
- for row in rows:
- d = dict(row)
-
- d.pop("stream_ordering", None)
- d.pop("topological_ordering", None)
- d.pop("processed", None)
-
- if "origin_server_ts" not in d:
- d["origin_server_ts"] = d.pop("ts", 0)
- else:
- d.pop("ts", 0)
-
- d.pop("prev_state", None)
- d.update(json.loads(d.pop("unrecognized_keys")))
-
- d["sender"] = d.pop("user_id")
-
- d["content"] = json.loads(d["content"])
-
- if "age_ts" not in d:
- # For compatibility
- d["age_ts"] = d.get("origin_server_ts", 0)
-
- d.setdefault("unsigned", {})["age_ts"] = d.pop("age_ts")
-
- outlier = d.pop("outlier", False)
-
- # d.pop("membership", None)
-
- d.pop("state_hash", None)
-
- d.pop("replaces_state", None)
-
- b = EventBuilder(d)
- b.internal_metadata.outlier = outlier
-
- events.append(b)
-
- for i, ev in enumerate(events):
- signatures = self._get_event_signatures_txn(
- txn, ev.event_id,
- )
-
- ev.signatures = {
- n: {
- k: encode_base64(v) for k, v in s.items()
- }
- for n, s in signatures.items()
- }
-
- hashes = self._get_event_content_hashes_txn(
- txn, ev.event_id,
- )
-
- ev.hashes = {
- k: encode_base64(v) for k, v in hashes.items()
- }
-
- prevs = self._get_prev_events_and_state(txn, ev.event_id)
-
- ev.prev_events = [
- (e_id, h)
- for e_id, h, is_state in prevs
- if is_state == 0
- ]
-
- # ev.auth_events = self._get_auth_events(txn, ev.event_id)
-
- hashes = dict(ev.auth_events)
-
- for e_id, hash in ev.prev_events:
- if e_id in hashes and not hash:
- hash.update(hashes[e_id])
- #
- # if hasattr(ev, "state_key"):
- # ev.prev_state = [
- # (e_id, h)
- # for e_id, h, is_state in prevs
- # if is_state == 1
- # ]
-
- return [e.build() for e in events]
-
-
-store = Store()
-
-
-# def get_key(server_name):
-# print "Getting keys for: %s" % (server_name,)
-# targets = []
-# if ":" in server_name:
-# target, port = server_name.split(":")
-# targets.append((target, int(port)))
-# try:
-# answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV")
-# for srv in answers:
-# targets.append((srv.target, srv.port))
-# except dns.resolver.NXDOMAIN:
-# targets.append((server_name, 8448))
-# except:
-# print "Failed to lookup keys for %s" % (server_name,)
-# return {}
-#
-# for target, port in targets:
-# url = "https://%s:%i/_matrix/key/v1" % (target, port)
-# try:
-# keys = json.load(urllib2.urlopen(url, timeout=2))
-# verify_keys = {}
-# for key_id, key_base64 in keys["verify_keys"].items():
-# verify_key = decode_verify_key_bytes(
-# key_id, decode_base64(key_base64)
-# )
-# verify_signed_json(keys, server_name, verify_key)
-# verify_keys[key_id] = verify_key
-# print "Got keys for: %s" % (server_name,)
-# return verify_keys
-# except urllib2.URLError:
-# pass
-# except urllib2.HTTPError:
-# pass
-# except httplib.HTTPException:
-# pass
-#
-# print "Failed to get keys for %s" % (server_name,)
-# return {}
-
-
-def reinsert_events(cursor, server_name, signing_key):
- print "Running delta: v10"
-
- cursor.executescript(delta_sql)
-
- cursor.execute(
- "SELECT * FROM events ORDER BY rowid ASC"
- )
-
- print "Getting events..."
-
- rows = store.cursor_to_dict(cursor)
-
- events = store._generate_event_json(cursor, rows)
-
- print "Got events from DB."
-
- algorithms = {
- "sha256": hashlib.sha256,
- }
-
- key_id = "%s:%s" % (signing_key.alg, signing_key.version)
- verify_key = signing_key.verify_key
- verify_key.alg = signing_key.alg
- verify_key.version = signing_key.version
-
- server_keys = {
- server_name: {
- key_id: verify_key
- }
- }
-
- i = 0
- N = len(events)
-
- for event in events:
- if i % 100 == 0:
- print "Processed: %d/%d events" % (i,N,)
- i += 1
-
- # for alg_name in event.hashes:
- # if check_event_content_hash(event, algorithms[alg_name]):
- # pass
- # else:
- # pass
- # print "FAIL content hash %s %s" % (alg_name, event.event_id, )
-
- have_own_correctly_signed = False
- for host, sigs in event.signatures.items():
- pruned = prune_event(event)
-
- for key_id in sigs:
- if host not in server_keys:
- server_keys[host] = {} # get_key(host)
- if key_id in server_keys[host]:
- try:
- verify_signed_json(
- pruned.get_pdu_json(),
- host,
- server_keys[host][key_id]
- )
-
- if host == server_name:
- have_own_correctly_signed = True
- except SignatureVerifyException:
- print "FAIL signature check %s %s" % (
- key_id, event.event_id
- )
-
- # TODO: Re sign with our own server key
- if not have_own_correctly_signed:
- sigs = compute_event_signature(event, server_name, signing_key)
- event.signatures.update(sigs)
-
- pruned = prune_event(event)
-
- for key_id in event.signatures[server_name]:
- verify_signed_json(
- pruned.get_pdu_json(),
- server_name,
- server_keys[server_name][key_id]
- )
-
- event_json = encode_canonical_json(
- event.get_dict()
- ).decode("UTF-8")
-
- metadata_json = encode_canonical_json(
- event.internal_metadata.get_dict()
- ).decode("UTF-8")
-
- store._simple_insert_txn(
- cursor,
- table="event_json",
- values={
- "event_id": event.event_id,
- "room_id": event.room_id,
- "internal_metadata": metadata_json,
- "json": event_json,
- },
- or_replace=True,
- )
-
-
-def main(database, server_name, signing_key):
- conn = sqlite3.connect(database)
- cursor = conn.cursor()
-
- # Do other deltas:
- cursor.execute("PRAGMA user_version")
- row = cursor.fetchone()
-
- if row and row[0]:
- user_version = row[0]
- # Run every version since after the current version.
- for v in range(user_version + 1, 10):
- print "Running delta: %d" % (v,)
- sql_script = read_schema("delta/v%d" % (v,))
- cursor.executescript(sql_script)
-
- reinsert_events(cursor, server_name, signing_key)
-
- conn.commit()
-
- print "Success!"
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
-
- parser.add_argument("database")
- parser.add_argument("server_name")
- parser.add_argument(
- "signing_key", type=argparse.FileType('r'),
- )
- args = parser.parse_args()
-
- signing_key = signedjson.key.read_signing_keys(args.signing_key)
-
- main(args.database, args.server_name, signing_key[0])
diff --git a/setup.cfg b/setup.cfg
index abb649958e..ba027c7d13 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,9 +3,6 @@ source-dir = docs/sphinx
build-dir = docs/build
all_files = 1
-[aliases]
-test = trial
-
[trial]
test_suite = tests
diff --git a/setup.py b/setup.py
index 8ad20df7cb..9d24761d44 100755
--- a/setup.py
+++ b/setup.py
@@ -16,7 +16,8 @@
import glob
import os
-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, Command
+import sys
here = os.path.abspath(os.path.dirname(__file__))
@@ -37,6 +38,39 @@ def exec_file(path_segments):
exec(code, result)
return result
+
+class Tox(Command):
+ user_options = [('tox-args=', 'a', "Arguments to pass to tox")]
+
+ def initialize_options(self):
+ self.tox_args = None
+
+ def finalize_options(self):
+ self.test_args = []
+ self.test_suite = True
+
+ def run(self):
+ #import here, cause outside the eggs aren't loaded
+ try:
+ import tox
+ except ImportError:
+ try:
+ self.distribution.fetch_build_eggs("tox")
+ import tox
+ except:
+ raise RuntimeError(
+ "The tests need 'tox' to run. Please install 'tox'."
+ )
+ import shlex
+ args = self.tox_args
+ if args:
+ args = shlex.split(self.tox_args)
+ else:
+ args = []
+ errno = tox.cmdline(args=args)
+ sys.exit(errno)
+
+
version = exec_file(("synapse", "__init__.py"))["__version__"]
dependencies = exec_file(("synapse", "python_dependencies.py"))
long_description = read_file(("README.rst",))
@@ -47,9 +81,10 @@ setup(
packages=find_packages(exclude=["tests", "tests.*"]),
description="Reference Synapse Home Server",
install_requires=dependencies['requirements'](include_conditional=True).keys(),
- dependency_links=dependencies["DEPENDENCY_LINKS"],
+ dependency_links=dependencies["DEPENDENCY_LINKS"].values(),
include_package_data=True,
zip_safe=False,
long_description=long_description,
scripts=["synctl"] + glob.glob("scripts/*"),
+ cmdclass={'test': Tox},
)
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 5853165a21..d85bb3dce0 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
""" This is a reference implementation of a Matrix home server.
"""
-__version__ = "0.10.0-rc1"
+__version__ = "0.10.0"
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 0a77a76cb8..df788230fa 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -20,7 +20,7 @@ from twisted.internet import defer
from synapse.api.constants import EventTypes, Membership, JoinRules
from synapse.api.errors import AuthError, Codes, SynapseError
from synapse.util.logutils import log_function
-from synapse.types import UserID
+from synapse.types import UserID, EventID
import logging
import pymacaroons
@@ -72,6 +72,14 @@ class Auth(object):
# FIXME
return True
+ creation_event = auth_events.get((EventTypes.Create, ""), None)
+
+ if not creation_event:
+ raise SynapseError(
+ 403,
+ "Room %r does not exist" % (event.room_id,)
+ )
+
# FIXME: Temp hack
if event.type == EventTypes.Aliases:
return True
@@ -98,7 +106,7 @@ class Auth(object):
self._check_power_levels(event, auth_events)
if event.type == EventTypes.Redaction:
- self._check_redaction(event, auth_events)
+ self.check_redaction(event, auth_events)
logger.debug("Allowing! %s", event)
except AuthError as e:
@@ -627,16 +635,35 @@ class Auth(object):
return True
- def _check_redaction(self, event, auth_events):
+ def check_redaction(self, event, auth_events):
+ """Check whether the event sender is allowed to redact the target event.
+
+ Returns:
+ True if the the sender is allowed to redact the target event if the
+ target event was created by them.
+ False if the sender is allowed to redact the target event with no
+ further checks.
+
+ Raises:
+ AuthError if the event sender is definitely not allowed to redact
+ the target event.
+ """
user_level = self._get_user_power_level(event.user_id, auth_events)
redact_level = self._get_named_level(auth_events, "redact", 50)
- if user_level < redact_level:
- raise AuthError(
- 403,
- "You don't have permission to redact events"
- )
+ if user_level > redact_level:
+ return False
+
+ redacter_domain = EventID.from_string(event.event_id).domain
+ redactee_domain = EventID.from_string(event.redacts).domain
+ if redacter_domain == redactee_domain:
+ return True
+
+ raise AuthError(
+ 403,
+ "You don't have permission to redact events"
+ )
def _check_power_levels(self, event, auth_events):
user_list = event.content.get("users", {})
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index f04493f92a..c23f853230 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -16,7 +16,7 @@
import sys
sys.dont_write_bytecode = True
-from synapse.python_dependencies import check_requirements
+from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS
if __name__ == '__main__':
check_requirements()
@@ -97,9 +97,25 @@ class SynapseHomeServer(HomeServer):
return JsonResource(self)
def build_resource_for_web_client(self):
- import syweb
- syweb_path = os.path.dirname(syweb.__file__)
- webclient_path = os.path.join(syweb_path, "webclient")
+ webclient_path = self.get_config().web_client_location
+ if not webclient_path:
+ try:
+ import syweb
+ except ImportError:
+ quit_with_error(
+ "Could not find a webclient.\n\n"
+ "Please either install the matrix-angular-sdk or configure\n"
+ "the location of the source to serve via the configuration\n"
+ "option `web_client_location`\n\n"
+ "To install the `matrix-angular-sdk` via pip, run:\n\n"
+ " pip install '%(dep)s'\n"
+ "\n"
+ "You can also disable hosting of the webclient via the\n"
+ "configuration option `web_client`\n"
+ % {"dep": DEPENDENCY_LINKS["matrix-angular-sdk"]}
+ )
+ syweb_path = os.path.dirname(syweb.__file__)
+ webclient_path = os.path.join(syweb_path, "webclient")
# GZip is disabled here due to
# https://twistedmatrix.com/trac/ticket/7678
# (It can stay enabled for the API resources: they call
@@ -259,11 +275,10 @@ class SynapseHomeServer(HomeServer):
def quit_with_error(error_string):
message_lines = error_string.split("\n")
- line_length = max([len(l) for l in message_lines]) + 2
+ line_length = max([len(l) for l in message_lines if len(l) < 80]) + 2
sys.stderr.write("*" * line_length + '\n')
for line in message_lines:
- if line.strip():
- sys.stderr.write(" %s\n" % (line.strip(),))
+ sys.stderr.write(" %s\n" % (line.rstrip(),))
sys.stderr.write("*" * line_length + '\n')
sys.exit(1)
@@ -326,7 +341,7 @@ def get_version_string():
)
).encode("ascii")
except Exception as e:
- logger.warn("Failed to check for git repository: %s", e)
+ logger.info("Failed to check for git repository: %s", e)
return ("Synapse/%s" % (synapse.__version__,)).encode("ascii")
@@ -388,7 +403,7 @@ def setup(config_options):
database_engine=database_engine,
)
- logger.info("Preparing database: %r...", config.database_config)
+ logger.info("Preparing database: %s...", config.database_config['name'])
try:
db_conn = database_engine.module.connect(
@@ -410,7 +425,7 @@ def setup(config_options):
)
sys.exit(1)
- logger.info("Database prepared in %r.", config.database_config)
+ logger.info("Database prepared in %s.", config.database_config['name'])
hs.start_listening()
diff --git a/synapse/config/__main__.py b/synapse/config/__main__.py
new file mode 100644
index 0000000000..f822d12036
--- /dev/null
+++ b/synapse/config/__main__.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if __name__ == "__main__":
+ import sys
+ from homeserver import HomeServerConfig
+
+ action = sys.argv[1]
+
+ if action == "read":
+ key = sys.argv[2]
+ config = HomeServerConfig.load_config("", sys.argv[3:])
+
+ print getattr(config, key)
+ sys.exit(0)
+ else:
+ sys.stderr.write("Unknown command %r\n" % (action,))
+ sys.exit(1)
diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index 73f6959959..8a75c48733 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -131,7 +131,8 @@ class Config(object):
"-c", "--config-path",
action="append",
metavar="CONFIG_FILE",
- help="Specify config file"
+ help="Specify config file. Can be given multiple times and"
+ " may specify directories containing *.yaml files."
)
config_parser.add_argument(
"--generate-config",
@@ -144,6 +145,13 @@ class Config(object):
help="Generate any missing key files then exit"
)
config_parser.add_argument(
+ "--keys-directory",
+ metavar="DIRECTORY",
+ help="Used with 'generate-*' options to specify where files such as"
+ " certs and signing keys should be stored in, unless explicitly"
+ " specified in the config."
+ )
+ config_parser.add_argument(
"-H", "--server-name",
help="The server name to generate a config file for"
)
@@ -151,16 +159,48 @@ class Config(object):
generate_keys = config_args.generate_keys
+ config_files = []
+ if config_args.config_path:
+ for config_path in config_args.config_path:
+ if os.path.isdir(config_path):
+ # We accept specifying directories as config paths, we search
+ # inside that directory for all files matching *.yaml, and then
+ # we apply them in *sorted* order.
+ files = []
+ for entry in os.listdir(config_path):
+ entry_path = os.path.join(config_path, entry)
+ if not os.path.isfile(entry_path):
+ print (
+ "Found subdirectory in config directory: %r. IGNORING."
+ ) % (entry_path, )
+ continue
+
+ if not entry.endswith(".yaml"):
+ print (
+ "Found file in config directory that does not"
+ " end in '.yaml': %r. IGNORING."
+ ) % (entry_path, )
+ continue
+
+ files.append(entry_path)
+
+ config_files.extend(sorted(files))
+ else:
+ config_files.append(config_path)
+
if config_args.generate_config:
- if not config_args.config_path:
+ if not config_files:
config_parser.error(
"Must supply a config file.\nA config file can be automatically"
" generated using \"--generate-config -H SERVER_NAME"
" -c CONFIG-FILE\""
)
- (config_path,) = config_args.config_path
+ (config_path,) = config_files
if not os.path.exists(config_path):
- config_dir_path = os.path.dirname(config_path)
+ if config_args.keys_directory:
+ config_dir_path = config_args.keys_directory
+ else:
+ config_dir_path = os.path.dirname(config_path)
config_dir_path = os.path.abspath(config_dir_path)
server_name = config_args.server_name
@@ -202,19 +242,22 @@ class Config(object):
obj.invoke_all("add_arguments", parser)
args = parser.parse_args(remaining_args)
- if not config_args.config_path:
+ if not config_files:
config_parser.error(
"Must supply a config file.\nA config file can be automatically"
" generated using \"--generate-config -H SERVER_NAME"
" -c CONFIG-FILE\""
)
- config_dir_path = os.path.dirname(config_args.config_path[-1])
+ if config_args.keys_directory:
+ config_dir_path = config_args.keys_directory
+ else:
+ config_dir_path = os.path.dirname(config_args.config_path[-1])
config_dir_path = os.path.abspath(config_dir_path)
specified_config = {}
- for config_path in config_args.config_path:
- yaml_config = cls.read_config_file(config_path)
+ for config_file in config_files:
+ yaml_config = cls.read_config_file(config_file)
specified_config.update(yaml_config)
server_name = specified_config["server_name"]
diff --git a/synapse/config/server.py b/synapse/config/server.py
index f9a3b5f15b..a03e55c223 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -22,6 +22,7 @@ class ServerConfig(Config):
self.server_name = config["server_name"]
self.pid_file = self.abspath(config.get("pid_file"))
self.web_client = config["web_client"]
+ self.web_client_location = config.get("web_client_location", None)
self.soft_file_limit = config["soft_file_limit"]
self.daemonize = config.get("daemonize")
self.print_pidfile = config.get("print_pidfile")
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index a692cdbe55..e251ab6af3 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -162,7 +162,9 @@ class Keyring(object):
def remove_deferreds(res, server_name, group_id):
server_to_gids[server_name].discard(group_id)
if not server_to_gids[server_name]:
- server_to_deferred.pop(server_name).callback(None)
+ d = server_to_deferred.pop(server_name, None)
+ if d:
+ d.callback(None)
return res
for g_id, deferred in deferreds.items():
@@ -200,8 +202,15 @@ class Keyring(object):
else:
break
- for server_name, deferred in server_to_deferred:
- self.key_downloads[server_name] = ObservableDeferred(deferred)
+ for server_name, deferred in server_to_deferred.items():
+ d = ObservableDeferred(deferred)
+ self.key_downloads[server_name] = d
+
+ def rm(r, server_name):
+ self.key_downloads.pop(server_name, None)
+ return r
+
+ d.addBoth(rm, server_name)
def get_server_verify_keys(self, group_id_to_group, group_id_to_deferred):
"""Takes a dict of KeyGroups and tries to find at least one key for
@@ -220,9 +229,8 @@ class Keyring(object):
merged_results = {}
missing_keys = {
- group.server_name: key_id
+ group.server_name: set(group.key_ids)
for group in group_id_to_group.values()
- for key_id in group.key_ids
}
for fn in key_fetch_fns:
@@ -279,16 +287,15 @@ class Keyring(object):
def get_keys_from_store(self, server_name_and_key_ids):
res = yield defer.gatherResults(
[
- self.store.get_server_verify_keys(server_name, key_ids)
+ self.store.get_server_verify_keys(
+ server_name, key_ids
+ ).addCallback(lambda ks, server: (server, ks), server_name)
for server_name, key_ids in server_name_and_key_ids
],
consumeErrors=True,
).addErrback(unwrapFirstError)
- defer.returnValue(dict(zip(
- [server_name for server_name, _ in server_name_and_key_ids],
- res
- )))
+ defer.returnValue(dict(res))
@defer.inlineCallbacks
def get_keys_from_perspectives(self, server_name_and_key_ids):
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index e91f1129db..60ac6617ae 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -15,7 +15,7 @@
from twisted.internet import defer
-from synapse.api.errors import LimitExceededError, SynapseError
+from synapse.api.errors import LimitExceededError, SynapseError, AuthError
from synapse.crypto.event_signing import add_hashes_and_signatures
from synapse.api.constants import Membership, EventTypes
from synapse.types import UserID, RoomAlias
@@ -107,6 +107,22 @@ class BaseHandler(object):
if not suppress_auth:
self.auth.check(event, auth_events=context.current_state)
+ if event.type == EventTypes.CanonicalAlias:
+ # Check the alias is acually valid (at this time at least)
+ room_alias_str = event.content.get("alias", None)
+ if room_alias_str:
+ room_alias = RoomAlias.from_string(room_alias_str)
+ directory_handler = self.hs.get_handlers().directory_handler
+ mapping = yield directory_handler.get_association(room_alias)
+
+ if mapping["room_id"] != event.room_id:
+ raise SynapseError(
+ 400,
+ "Room alias %s does not point to the room" % (
+ room_alias_str,
+ )
+ )
+
(event_stream_id, max_stream_id) = yield self.store.persist_event(
event, context=context
)
@@ -130,20 +146,19 @@ class BaseHandler(object):
returned_invite.signatures
)
- if event.type == EventTypes.CanonicalAlias:
- # Check the alias is acually valid (at this time at least)
- room_alias_str = event.content.get("alias", None)
- if room_alias_str:
- room_alias = RoomAlias.from_string(room_alias_str)
- directory_handler = self.hs.get_handlers().directory_handler
- mapping = yield directory_handler.get_association(room_alias)
-
- if mapping["room_id"] != event.room_id:
- raise SynapseError(
- 400,
- "Room alias %s does not point to the room" % (
- room_alias_str,
- )
+ if event.type == EventTypes.Redaction:
+ if self.auth.check_redaction(event, auth_events=context.current_state):
+ original_event = yield self.store.get_event(
+ event.redacts,
+ check_redacted=False,
+ get_prev_content=False,
+ allow_rejected=False,
+ allow_none=False
+ )
+ if event.user_id != original_event.user_id:
+ raise AuthError(
+ 403,
+ "You don't have permission to redact events"
)
destinations = set(extra_destinations)
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 1ab19cd1a6..59f687e0f1 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -324,7 +324,7 @@ class AuthHandler(BaseHandler):
def _check_password(self, user_id, password, stored_hash):
"""Checks that user_id has passed password, raises LoginError if not."""
- if not bcrypt.checkpw(password, stored_hash):
+ if not self.validate_hash(password, stored_hash):
logger.warn("Failed password login for user %s", user_id)
raise LoginError(403, "", errcode=Codes.FORBIDDEN)
@@ -369,7 +369,7 @@ class AuthHandler(BaseHandler):
@defer.inlineCallbacks
def set_password(self, user_id, newpassword):
- password_hash = bcrypt.hashpw(newpassword, bcrypt.gensalt())
+ password_hash = self.hash(newpassword)
yield self.store.user_set_password_hash(user_id, password_hash)
yield self.store.user_delete_access_tokens(user_id)
@@ -391,3 +391,26 @@ class AuthHandler(BaseHandler):
def _remove_session(self, session):
logger.debug("Removing session %s", session)
del self.sessions[session["id"]]
+
+ def hash(self, password):
+ """Computes a secure hash of password.
+
+ Args:
+ password (str): Password to hash.
+
+ Returns:
+ Hashed password (str).
+ """
+ return bcrypt.hashpw(password, bcrypt.gensalt())
+
+ def validate_hash(self, password, stored_hash):
+ """Validates that self.hash(password) == stored_hash.
+
+ Args:
+ password (str): Password to hash.
+ stored_hash (str): Expected hash value.
+
+ Returns:
+ Whether self.hash(password) == stored_hash (bool).
+ """
+ return bcrypt.checkpw(password, stored_hash)
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index f9ca2f8634..891502c04f 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -49,7 +49,12 @@ class EventStreamHandler(BaseHandler):
@defer.inlineCallbacks
@log_function
def get_stream(self, auth_user_id, pagin_config, timeout=0,
- as_client_event=True, affect_presence=True):
+ as_client_event=True, affect_presence=True,
+ only_room_events=False):
+ """Fetches the events stream for a given user.
+
+ If `only_room_events` is `True` only room events will be returned.
+ """
auth_user = UserID.from_string(auth_user_id)
try:
@@ -89,7 +94,8 @@ class EventStreamHandler(BaseHandler):
timeout = random.randint(int(timeout*0.9), int(timeout*1.1))
events, tokens = yield self.notifier.get_events_for(
- auth_user, room_ids, pagin_config, timeout
+ auth_user, room_ids, pagin_config, timeout,
+ only_room_events=only_room_events
)
time_now = self.clock.time_msec()
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 56d125f753..ef4081e3fe 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -25,7 +25,6 @@ import synapse.util.stringutils as stringutils
from synapse.util.async import run_on_reactor
from synapse.http.client import CaptchaServerHttpClient
-import bcrypt
import logging
import urllib
@@ -82,7 +81,7 @@ class RegistrationHandler(BaseHandler):
yield run_on_reactor()
password_hash = None
if password:
- password_hash = bcrypt.hashpw(password, bcrypt.gensalt())
+ password_hash = self.auth_handler().hash(password)
if localpart:
yield self.check_username(localpart)
diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index 026bd2b9d4..d7096aab8c 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -204,15 +204,11 @@ class TypingNotificationHandler(BaseHandler):
)
def _push_update_local(self, room_id, user, typing):
- if room_id not in self._room_serials:
- self._room_serials[room_id] = 0
- self._room_typing[room_id] = set()
-
- room_set = self._room_typing[room_id]
+ room_set = self._room_typing.setdefault(room_id, set())
if typing:
room_set.add(user)
- elif user in room_set:
- room_set.remove(user)
+ else:
+ room_set.discard(user)
self._latest_room_serial += 1
self._room_serials[room_id] = self._latest_room_serial
@@ -260,8 +256,8 @@ class TypingNotificationEventSource(object):
)
events = []
- for room_id in handler._room_serials:
- if room_id not in joined_room_ids:
+ for room_id in joined_room_ids:
+ if room_id not in handler._room_serials:
continue
if handler._room_serials[room_id] <= from_key:
continue
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index d7bcad8a8a..943d637459 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -17,7 +17,7 @@
from __future__ import absolute_import
import logging
-from resource import getrusage, getpagesize, RUSAGE_SELF
+from resource import getrusage, RUSAGE_SELF
import functools
import os
import stat
@@ -100,7 +100,6 @@ def render_all():
# process resource usage
rusage = None
-PAGE_SIZE = getpagesize()
def update_resource_metrics():
@@ -113,8 +112,8 @@ resource_metrics = get_metrics_for("process.resource")
resource_metrics.register_callback("utime", lambda: rusage.ru_utime * 1000)
resource_metrics.register_callback("stime", lambda: rusage.ru_stime * 1000)
-# pages
-resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * PAGE_SIZE)
+# kilobytes
+resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * 1024)
TYPES = {
stat.S_IFSOCK: "SOCK",
@@ -131,6 +130,10 @@ def _process_fds():
counts = {(k,): 0 for k in TYPES.values()}
counts[("other",)] = 0
+ # Not every OS will have a /proc/self/fd directory
+ if not os.path.exists("/proc/self/fd"):
+ return counts
+
for fd in os.listdir("/proc/self/fd"):
try:
s = os.stat("/proc/self/fd/%s" % (fd))
diff --git a/synapse/notifier.py b/synapse/notifier.py
index dbd8efe9fb..f998fc83bf 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -328,10 +328,13 @@ class Notifier(object):
defer.returnValue(result)
@defer.inlineCallbacks
- def get_events_for(self, user, rooms, pagination_config, timeout):
+ def get_events_for(self, user, rooms, pagination_config, timeout,
+ only_room_events=False):
""" For the given user and rooms, return any new events for them. If
there are no new events wait for up to `timeout` milliseconds for any
new events to happen before returning.
+
+ If `only_room_events` is `True` only room events will be returned.
"""
from_token = pagination_config.from_token
if not from_token:
@@ -352,6 +355,8 @@ class Notifier(object):
after_id = getattr(after_token, keyname)
if before_id == after_id:
continue
+ if only_room_events and name != "room":
+ continue
new_events, new_key = yield source.get_new_events_for_user(
user, getattr(from_token, keyname), limit,
)
diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py
index 13002e0db4..f1952b5a0f 100644
--- a/synapse/push/__init__.py
+++ b/synapse/push/__init__.py
@@ -249,7 +249,9 @@ class Pusher(object):
# we fail to dispatch the push)
config = PaginationConfig(from_token=None, limit='1')
chunk = yield self.evStreamHandler.get_stream(
- self.user_name, config, timeout=0)
+ self.user_name, config, timeout=0, affect_presence=False,
+ only_room_events=True
+ )
self.last_token = chunk['end']
self.store.update_pusher_last_token(
self.app_id, self.pushkey, self.user_name, self.last_token
@@ -280,8 +282,8 @@ class Pusher(object):
config = PaginationConfig(from_token=from_tok, limit='1')
timeout = (300 + random.randint(-60, 60)) * 1000
chunk = yield self.evStreamHandler.get_stream(
- self.user_name, config,
- timeout=timeout, affect_presence=False
+ self.user_name, config, timeout=timeout, affect_presence=False,
+ only_room_events=True
)
# limiting to 1 may get 1 event plus 1 presence event, so
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 4c07aa07aa..795ef27182 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -46,8 +46,8 @@ CONDITIONAL_REQUIREMENTS = {
def requirements(config=None, include_conditional=False):
reqs = REQUIREMENTS.copy()
- for key, req in CONDITIONAL_REQUIREMENTS.items():
- if (config and getattr(config, key)) or include_conditional:
+ if include_conditional:
+ for _, req in CONDITIONAL_REQUIREMENTS.items():
reqs.update(req)
return reqs
@@ -55,13 +55,8 @@ def requirements(config=None, include_conditional=False):
def github_link(project, version, egg):
return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)
-DEPENDENCY_LINKS = [
- github_link(
- project="matrix-org/matrix-angular-sdk",
- version="v0.6.6",
- egg="matrix_angular_sdk-0.6.6",
- ),
-]
+DEPENDENCY_LINKS = {
+}
class MissingRequirementError(Exception):
@@ -129,7 +124,7 @@ def check_requirements(config=None):
def list_requirements():
result = []
linked = []
- for link in DEPENDENCY_LINKS:
+ for link in DEPENDENCY_LINKS.values():
egg = link.split("#egg=")[1]
linked.append(egg.split('-')[0])
result.append(link)
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index b5edffdb60..4692ba413c 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -96,6 +96,7 @@ class ThreepidRestServlet(RestServlet):
self.hs = hs
self.identity_handler = hs.get_handlers().identity_handler
self.auth = hs.get_auth()
+ self.auth_handler = hs.get_handlers().auth_handler
@defer.inlineCallbacks
def on_GET(self, request):
diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index 4e21527c3d..b2aeb8c909 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -33,6 +33,8 @@ import os
import cgi
import logging
+import urllib
+import urlparse
logger = logging.getLogger(__name__)
@@ -42,10 +44,13 @@ def parse_media_id(request):
# This allows users to append e.g. /test.png to the URL. Useful for
# clients that parse the URL to see content type.
server_name, media_id = request.postpath[:2]
- if len(request.postpath) > 2 and is_ascii(request.postpath[-1]):
- return server_name, media_id, request.postpath[-1]
- else:
- return server_name, media_id, None
+ file_name = None
+ if len(request.postpath) > 2:
+ try:
+ file_name = urlparse.unquote(request.postpath[-1]).decode("utf-8")
+ except UnicodeDecodeError:
+ pass
+ return server_name, media_id, file_name
except:
raise SynapseError(
404,
@@ -140,9 +145,26 @@ class BaseMediaResource(Resource):
content_disposition = headers.get("Content-Disposition", None)
if content_disposition:
_, params = cgi.parse_header(content_disposition[0],)
- upload_name = params.get("filename", None)
- if upload_name and not is_ascii(upload_name):
- upload_name = None
+ upload_name = None
+
+ # First check if there is a valid UTF-8 filename
+ upload_name_utf8 = params.get("filename*", None)
+ if upload_name_utf8:
+ if upload_name_utf8.lower().startswith("utf-8''"):
+ upload_name = upload_name_utf8[7:]
+
+ # If there isn't check for an ascii name.
+ if not upload_name:
+ upload_name_ascii = params.get("filename", None)
+ if upload_name_ascii and is_ascii(upload_name_ascii):
+ upload_name = upload_name_ascii
+
+ if upload_name:
+ upload_name = urlparse.unquote(upload_name)
+ try:
+ upload_name = upload_name.decode("utf-8")
+ except UnicodeDecodeError:
+ upload_name = None
else:
upload_name = None
@@ -181,10 +203,20 @@ class BaseMediaResource(Resource):
if os.path.isfile(file_path):
request.setHeader(b"Content-Type", media_type.encode("UTF-8"))
if upload_name:
- request.setHeader(
- b"Content-Disposition",
- b"inline; filename=%s" % (upload_name.encode("utf-8"),),
- )
+ if is_ascii(upload_name):
+ request.setHeader(
+ b"Content-Disposition",
+ b"inline; filename=%s" % (
+ urllib.quote(upload_name.encode("utf-8")),
+ ),
+ )
+ else:
+ request.setHeader(
+ b"Content-Disposition",
+ b"inline; filename*=utf-8''%s" % (
+ urllib.quote(upload_name.encode("utf-8")),
+ ),
+ )
# cache for at least a day.
# XXX: we might want to turn this off for data we don't want to
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index 439d5a30a8..6abaf56b25 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -15,7 +15,7 @@
from synapse.http.server import respond_with_json, request_handler
-from synapse.util.stringutils import random_string, is_ascii
+from synapse.util.stringutils import random_string
from synapse.api.errors import SynapseError
from twisted.web.server import NOT_DONE_YET
@@ -86,9 +86,13 @@ class UploadResource(BaseMediaResource):
upload_name = request.args.get("filename", None)
if upload_name:
- upload_name = upload_name[0]
- if upload_name and not is_ascii(upload_name):
- raise SynapseError(400, "filename must be ascii")
+ try:
+ upload_name = upload_name[0].decode('UTF-8')
+ except UnicodeDecodeError:
+ raise SynapseError(
+ msg="Invalid UTF-8 filename parameter: %r" % (upload_name),
+ code=400,
+ )
headers = request.requestHeaders
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index ce71389f02..495ef087c9 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -167,7 +167,7 @@ class SQLBaseStore(object):
self._get_event_cache = Cache("*getEvent*", keylen=3, lru=True,
max_entries=hs.config.event_cache_size)
- self._state_group_cache = DictionaryCache("*stateGroupCache*", 100000)
+ self._state_group_cache = DictionaryCache("*stateGroupCache*", 2000)
self._event_fetch_lock = threading.Condition()
self._event_fetch_list = []
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index bc90e17c63..989ad340b0 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -331,7 +331,10 @@ class EventFederationStore(SQLBaseStore):
txn.executemany(
query,
- [(ev.event_id, ev.room_id, ev.event_id) for ev in events]
+ [
+ (ev.event_id, ev.room_id, ev.event_id) for ev in events
+ if not ev.internal_metadata.is_outlier()
+ ]
)
query = (
@@ -358,7 +361,10 @@ class EventFederationStore(SQLBaseStore):
)
txn.executemany(
query,
- [(ev.event_id, ev.room_id) for ev in events]
+ [
+ (ev.event_id, ev.room_id) for ev in events
+ if not ev.internal_metadata.is_outlier()
+ ]
)
for room_id in events_by_room:
diff --git a/synapse/storage/schema/delta/23/drop_state_index.sql b/synapse/storage/schema/delta/23/drop_state_index.sql
new file mode 100644
index 0000000000..07d0ea5cb2
--- /dev/null
+++ b/synapse/storage/schema/delta/23/drop_state_index.sql
@@ -0,0 +1,16 @@
+/* Copyright 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+DROP INDEX IF EXISTS state_groups_state_tuple;
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index c9110e6304..9630efcfcc 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -403,8 +403,15 @@ class StateStore(SQLBaseStore):
state_dict = results[group]
for event_id in state_ids:
- state_event = state_events[event_id]
- state_dict[(state_event.type, state_event.state_key)] = state_event
+ try:
+ state_event = state_events[event_id]
+ state_dict[(state_event.type, state_event.state_key)] = state_event
+ except KeyError:
+ # Hmm. So we do don't have that state event? Interesting.
+ logger.warn(
+ "Can't find state event %r for state group %r",
+ event_id, group,
+ )
self._state_group_cache.update(
cache_seq_num,
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index 7a1e96af37..f3a36340e4 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -38,6 +38,8 @@ def random_string_with_symbols(length):
def is_ascii(s):
try:
s.encode("ascii")
+ except UnicodeEncodeError:
+ return False
except UnicodeDecodeError:
return False
else:
diff --git a/tests/test_state.py b/tests/test_state.py
index 5845358754..55f37c521f 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -204,8 +204,8 @@ class StateTestCase(unittest.TestCase):
nodes={
"START": DictObj(
type=EventTypes.Create,
- state_key="creator",
- content={"membership": "@user_id:example.com"},
+ state_key="",
+ content={"creator": "@user_id:example.com"},
depth=1,
),
"A": DictObj(
@@ -259,8 +259,8 @@ class StateTestCase(unittest.TestCase):
nodes={
"START": DictObj(
type=EventTypes.Create,
- state_key="creator",
- content={"membership": "@user_id:example.com"},
+ state_key="",
+ content={"creator": "@user_id:example.com"},
depth=1,
),
"A": DictObj(
@@ -432,13 +432,19 @@ class StateTestCase(unittest.TestCase):
def test_resolve_message_conflict(self):
event = create_event(type="test_message", name="event")
+ creation = create_event(
+ type=EventTypes.Create, state_key=""
+ )
+
old_state_1 = [
+ creation,
create_event(type="test1", state_key="1"),
create_event(type="test1", state_key="2"),
create_event(type="test2", state_key=""),
]
old_state_2 = [
+ creation,
create_event(type="test1", state_key="1"),
create_event(type="test3", state_key="2"),
create_event(type="test4", state_key=""),
@@ -446,7 +452,7 @@ class StateTestCase(unittest.TestCase):
context = yield self._get_context(event, old_state_1, old_state_2)
- self.assertEqual(len(context.current_state), 5)
+ self.assertEqual(len(context.current_state), 6)
self.assertIsNone(context.state_group)
@@ -454,13 +460,19 @@ class StateTestCase(unittest.TestCase):
def test_resolve_state_conflict(self):
event = create_event(type="test4", state_key="", name="event")
+ creation = create_event(
+ type=EventTypes.Create, state_key=""
+ )
+
old_state_1 = [
+ creation,
create_event(type="test1", state_key="1"),
create_event(type="test1", state_key="2"),
create_event(type="test2", state_key=""),
]
old_state_2 = [
+ creation,
create_event(type="test1", state_key="1"),
create_event(type="test3", state_key="2"),
create_event(type="test4", state_key=""),
@@ -468,7 +480,7 @@ class StateTestCase(unittest.TestCase):
context = yield self._get_context(event, old_state_1, old_state_2)
- self.assertEqual(len(context.current_state), 5)
+ self.assertEqual(len(context.current_state), 6)
self.assertIsNone(context.state_group)
@@ -484,36 +496,45 @@ class StateTestCase(unittest.TestCase):
}
)
+ creation = create_event(
+ type=EventTypes.Create, state_key="",
+ content={"creator": "@foo:bar"}
+ )
+
old_state_1 = [
+ creation,
member_event,
create_event(type="test1", state_key="1", depth=1),
]
old_state_2 = [
+ creation,
member_event,
create_event(type="test1", state_key="1", depth=2),
]
context = yield self._get_context(event, old_state_1, old_state_2)
- self.assertEqual(old_state_2[1], context.current_state[("test1", "1")])
+ self.assertEqual(old_state_2[2], context.current_state[("test1", "1")])
# Reverse the depth to make sure we are actually using the depths
# during state resolution.
old_state_1 = [
+ creation,
member_event,
create_event(type="test1", state_key="1", depth=2),
]
old_state_2 = [
+ creation,
member_event,
create_event(type="test1", state_key="1", depth=1),
]
context = yield self._get_context(event, old_state_1, old_state_2)
- self.assertEqual(old_state_1[1], context.current_state[("test1", "1")])
+ self.assertEqual(old_state_1[2], context.current_state[("test1", "1")])
def _get_context(self, event, old_state_1, old_state_2):
group_name_1 = "group_name_1"
diff --git a/tests/utils.py b/tests/utils.py
index 3766a994f2..dd19a16fc7 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -27,6 +27,7 @@ from twisted.enterprise.adbapi import ConnectionPool
from collections import namedtuple
from mock import patch, Mock
+import hashlib
import urllib
import urlparse
@@ -67,6 +68,18 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
**kargs
)
+ # bcrypt is far too slow to be doing in unit tests
+ def swap_out_hash_for_testing(old_build_handlers):
+ def build_handlers():
+ handlers = old_build_handlers()
+ auth_handler = handlers.auth_handler
+ auth_handler.hash = lambda p: hashlib.md5(p).hexdigest()
+ auth_handler.validate_hash = lambda p, h: hashlib.md5(p).hexdigest() == h
+ return handlers
+ return build_handlers
+
+ hs.build_handlers = swap_out_hash_for_testing(hs.build_handlers)
+
defer.returnValue(hs)
diff --git a/tox.ini b/tox.ini
index 58cf9613cb..a69948484f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,10 +9,8 @@ deps =
setenv =
PYTHONDONTWRITEBYTECODE = no_byte_code
commands =
- coverage run --source=./synapse {envbindir}/trial tests
+ coverage run --source=synapse {envbindir}/trial {posargs:tests}
coverage report -m
-install_command =
- pip install --process-dependency-links --pre {opts} {packages}
[testenv:packaging]
deps =
|
