diff options
| -rw-r--r-- | changelog.d/4435.bugfix | 1 | ||||
| -rw-r--r-- | changelog.d/4444.misc | 1 | ||||
| -rw-r--r-- | changelog.d/4458.misc | 1 | ||||
| -rw-r--r-- | changelog.d/4459.misc | 1 | ||||
| -rw-r--r-- | changelog.d/4460.bugfix | 1 | ||||
| -rw-r--r-- | changelog.d/4464.misc | 1 | ||||
| -rwxr-xr-x | debian/build_virtualenv | 34 | ||||
| -rw-r--r-- | debian/homeserver.yaml | 614 | ||||
| -rw-r--r-- | debian/install | 1 | ||||
| -rw-r--r-- | synapse/api/auth.py | 4 | ||||
| -rw-r--r-- | synapse/config/server.py | 15 | ||||
| -rw-r--r-- | synapse/http/client.py | 5 | ||||
| -rw-r--r-- | synapse/http/federation/matrix_federation_agent.py | 3 | ||||
| -rw-r--r-- | synapse/rest/client/v2_alpha/register.py | 7 | ||||
| -rw-r--r-- | synapse/storage/_base.py | 33 | ||||
| -rw-r--r-- | tests/http/federation/test_matrix_federation_agent.py | 97 | ||||
| -rw-r--r-- | tests/utils.py | 4 |
17 files changed, 177 insertions, 646 deletions
diff --git a/changelog.d/4435.bugfix b/changelog.d/4435.bugfix new file mode 100644 index 0000000000..4ea9a5df02 --- /dev/null +++ b/changelog.d/4435.bugfix @@ -0,0 +1 @@ +Fix None guard in calling config.server.is_threepid_reserved diff --git a/changelog.d/4444.misc b/changelog.d/4444.misc new file mode 100644 index 0000000000..1be84188c6 --- /dev/null +++ b/changelog.d/4444.misc @@ -0,0 +1 @@ +Generate the debian config during build diff --git a/changelog.d/4458.misc b/changelog.d/4458.misc new file mode 100644 index 0000000000..8b3bc94a34 --- /dev/null +++ b/changelog.d/4458.misc @@ -0,0 +1 @@ +Clarify documentation for the `public_baseurl` config param diff --git a/changelog.d/4459.misc b/changelog.d/4459.misc new file mode 100644 index 0000000000..58130b6190 --- /dev/null +++ b/changelog.d/4459.misc @@ -0,0 +1 @@ +Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. diff --git a/changelog.d/4460.bugfix b/changelog.d/4460.bugfix new file mode 100644 index 0000000000..8c5d5b4e0e --- /dev/null +++ b/changelog.d/4460.bugfix @@ -0,0 +1 @@ +Fix UnboundLocalError in post_urlencoded_get_json diff --git a/changelog.d/4464.misc b/changelog.d/4464.misc new file mode 100644 index 0000000000..9a51434755 --- /dev/null +++ b/changelog.d/4464.misc @@ -0,0 +1 @@ +Move SRV logic into the Agent layer diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 8b51b9e074..bead3ebc6e 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -45,6 +45,10 @@ dh_virtualenv \ --extra-pip-arg="--compile" \ --extras="all" +PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" +VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" +TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python" + # we copy the tests to a temporary directory so that we can put them on the # PYTHONPATH without putting the uninstalled synapse on the pythonpath. tmpdir=`mktemp -d` @@ -53,8 +57,34 @@ trap "rm -r $tmpdir" EXIT cp -r tests "$tmpdir" PYTHONPATH="$tmpdir" \ - debian/matrix-synapse-py3/opt/venvs/matrix-synapse/bin/python \ - -B -m twisted.trial --reporter=text -j2 tests + "${TARGET_PYTHON}" -B -m twisted.trial --reporter=text -j2 tests + +# build the config file +"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_config" \ + --config-dir="/etc/matrix-synapse" \ + --data-dir="/var/lib/matrix-synapse" | + perl -pe ' +# tweak the paths to the tls certs and signing keys +/^tls_.*_path:/ and s/SERVERNAME/homeserver/; +/^signing_key_path:/ and s/SERVERNAME/homeserver/; + +# tweak the pid file location +/^pid_file:/ and s#:.*#: "/var/run/matrix-synapse.pid"#; + +# tweak the path to the log config +/^log_config:/ and s/SERVERNAME\.log\.config/log.yaml/; + +# tweak the path to the media store +/^media_store_path:/ and s#/media_store#/media#; + +# remove the server_name setting, which is set in a separate file +/^server_name:/ and $_ = "#\n# This is set in /etc/matrix-synapse/conf.d/server_name.yaml for Debian installations.\n# $_"; + +# remove the report_stats setting, which is set in a separate file +/^# report_stats:/ and $_ = ""; + +' > "${PACKAGE_BUILD_DIR}/etc/matrix-synapse/homeserver.yaml" + # add a dependency on the right version of python to substvars. PYPKG=`basename $SNAKE` diff --git a/debian/homeserver.yaml b/debian/homeserver.yaml deleted file mode 100644 index 0bb2d22a95..0000000000 --- a/debian/homeserver.yaml +++ /dev/null @@ -1,614 +0,0 @@ -# vim:ft=yaml -# PEM encoded X509 certificate for TLS. -# You can replace the self-signed certificate that synapse -# autogenerates on launch with your own SSL certificate + key pair -# if you like. Any required intermediary certificates can be -# appended after the primary certificate in hierarchical order. -tls_certificate_path: "/etc/matrix-synapse/homeserver.tls.crt" - -# PEM encoded private key for TLS -tls_private_key_path: "/etc/matrix-synapse/homeserver.tls.key" - -# Don't bind to the https port -no_tls: False - -# List of allowed TLS fingerprints for this server to publish along -# with the signing keys for this server. Other matrix servers that -# make HTTPS requests to this server will check that the TLS -# certificates returned by this server match one of the fingerprints. -# -# Synapse automatically adds the fingerprint of its own certificate -# to the list. So if federation traffic is handled directly by synapse -# then no modification to the list is required. -# -# If synapse is run behind a load balancer that handles the TLS then it -# will be necessary to add the fingerprints of the certificates used by -# the loadbalancers to this list if they are different to the one -# synapse is using. -# -# Homeservers are permitted to cache the list of TLS fingerprints -# returned in the key responses up to the "valid_until_ts" returned in -# key. It may be necessary to publish the fingerprints of a new -# certificate and wait until the "valid_until_ts" of the previous key -# responses have passed before deploying it. -# -# You can calculate a fingerprint from a given TLS listener via: -# openssl s_client -connect $host:$port < /dev/null 2> /dev/null | -# openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '=' -# or by checking matrix.org/federationtester/api/report?server_name=$host -# -tls_fingerprints: [] -# tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}] - - -## Server ## - -# When running as a daemon, the file to store the pid in -pid_file: "/var/run/matrix-synapse.pid" - -# CPU affinity mask. Setting this restricts the CPUs on which the -# process will be scheduled. It is represented as a bitmask, with the -# lowest order bit corresponding to the first logical CPU and the -# highest order bit corresponding to the last logical CPU. Not all CPUs -# may exist on a given system but a mask may specify more CPUs than are -# present. -# -# For example: -# 0x00000001 is processor #0, -# 0x00000003 is processors #0 and #1, -# 0xFFFFFFFF is all processors (#0 through #31). -# -# Pinning a Python process to a single CPU is desirable, because Python -# is inherently single-threaded due to the GIL, and can suffer a -# 30-40% slowdown due to cache blow-out and thread context switching -# if the scheduler happens to schedule the underlying threads across -# different cores. See -# https://www.mirantis.com/blog/improve-performance-python-programs-restricting-single-cpu/. -# -# cpu_affinity: 0xFFFFFFFF - -# The path to the web client which will be served at /_matrix/client/ -# if 'webclient' is configured under the 'listeners' configuration. -# -# web_client_location: "/path/to/web/root" - -# The public-facing base URL for the client API (not including _matrix/...) -# public_baseurl: https://example.com:8448/ - -# Set the soft limit on the number of file descriptors synapse can use -# Zero is used to indicate synapse should set the soft limit to the -# hard limit. -soft_file_limit: 0 - -# The GC threshold parameters to pass to `gc.set_threshold`, if defined -# gc_thresholds: [700, 10, 10] - -# Set the limit on the returned events in the timeline in the get -# and sync operations. The default value is -1, means no upper limit. -# filter_timeline_limit: 5000 - -# Whether room invites to users on this server should be blocked -# (except those sent by local server admins). The default is False. -# block_non_admin_invites: True - -# Restrict federation to the following whitelist of domains. -# N.B. we recommend also firewalling your federation listener to limit -# inbound federation traffic as early as possible, rather than relying -# purely on this application-layer restriction. If not specified, the -# default is to whitelist everything. -# -# federation_domain_whitelist: -# - lon.example.com -# - nyc.example.com -# - syd.example.com - -# List of ports that Synapse should listen on, their purpose and their -# configuration. -listeners: - # Main HTTPS listener - # For when matrix traffic is sent directly to synapse. - - - # The port to listen for HTTPS requests on. - port: 8448 - - # Local addresses to listen on. - # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6 - # addresses by default. For most other OSes, this will only listen - # on IPv6. - bind_addresses: - - '::' - - '0.0.0.0' - - # This is a 'http' listener, allows us to specify 'resources'. - type: http - - tls: true - - # Use the X-Forwarded-For (XFF) header as the client IP and not the - # actual client IP. - x_forwarded: false - - # List of HTTP resources to serve on this listener. - resources: - - - # List of resources to host on this listener. - names: - - client # The client-server APIs, both v1 and v2 - - webclient # The bundled webclient. - - # Should synapse compress HTTP responses to clients that support it? - # This should be disabled if running synapse behind a load balancer - # that can do automatic compression. - compress: true - - - names: [federation] # Federation APIs - compress: false - - # optional list of additional endpoints which can be loaded via - # dynamic modules - # additional_resources: - # "/_matrix/my/custom/endpoint": - # module: my_module.CustomRequestHandler - # config: {} - - # Unsecure HTTP listener, - # For when matrix traffic passes through loadbalancer that unwraps TLS. - - port: 8008 - tls: false - bind_addresses: ['::', '0.0.0.0'] - type: http - - x_forwarded: false - - resources: - - names: [client, webclient] - compress: true - - names: [federation] - compress: false - - # Turn on the twisted ssh manhole service on localhost on the given - # port. - # - port: 9000 - # bind_addresses: ['::1', '127.0.0.1'] - # type: manhole - - -# Database configuration -database: - # The database engine name - name: "sqlite3" - # Arguments to pass to the engine - args: - # Path to the database - database: "/var/lib/matrix-synapse/homeserver.db" - -# Number of events to cache in memory. -event_cache_size: "10K" - - -# A yaml python logging config file -log_config: "/etc/matrix-synapse/log.yaml" - - - -## Ratelimiting ## - -# Number of messages a client can send per second -rc_messages_per_second: 0.2 - -# Number of message a client can send before being throttled -rc_message_burst_count: 10.0 - -# The federation window size in milliseconds -federation_rc_window_size: 1000 - -# The number of federation requests from a single server in a window -# before the server will delay processing the request. -federation_rc_sleep_limit: 10 - -# The duration in milliseconds to delay processing events from -# remote servers by if they go over the sleep limit. -federation_rc_sleep_delay: 500 - -# The maximum number of concurrent federation requests allowed -# from a single server -federation_rc_reject_limit: 50 - -# The number of federation requests to concurrently process from a -# single server -federation_rc_concurrent: 3 - - - -# Directory where uploaded images and attachments are stored. -media_store_path: "/var/lib/matrix-synapse/media" - -# Media storage providers allow media to be stored in different -# locations. -# media_storage_providers: -# - module: file_system -# # Whether to write new local files. -# store_local: false -# # Whether to write new remote media -# store_remote: false -# # Whether to block upload requests waiting for write to this -# # provider to complete -# store_synchronous: false -# config: -# directory: /mnt/some/other/directory - -# Directory where in-progress uploads are stored. -uploads_path: "/var/lib/matrix-synapse/uploads" - -# The largest allowed upload size in bytes -max_upload_size: "10M" - -# Maximum number of pixels that will be thumbnailed -max_image_pixels: "32M" - -# Whether to generate new thumbnails on the fly to precisely match -# the resolution requested by the client. If true then whenever -# a new resolution is requested by the client the server will -# generate a new thumbnail. If false the server will pick a thumbnail -# from a precalculated list. -dynamic_thumbnails: false - -# List of thumbnail to precalculate when an image is uploaded. -thumbnail_sizes: -- width: 32 - height: 32 - method: crop -- width: 96 - height: 96 - method: crop -- width: 320 - height: 240 - method: scale -- width: 640 - height: 480 - method: scale -- width: 800 - height: 600 - method: scale - -# Is the preview URL API enabled? If enabled, you *must* specify -# an explicit url_preview_ip_range_blacklist of IPs that the spider is -# denied from accessing. -url_preview_enabled: False - -# List of IP address CIDR ranges that the URL preview spider is denied -# from accessing. There are no defaults: you must explicitly -# specify a list for URL previewing to work. You should specify any -# internal services in your network that you do not want synapse to try -# to connect to, otherwise anyone in any Matrix room could cause your -# synapse to issue arbitrary GET requests to your internal services, -# causing serious security issues. -# -# url_preview_ip_range_blacklist: -# - '127.0.0.0/8' -# - '10.0.0.0/8' -# - '172.16.0.0/12' -# - '192.168.0.0/16' -# - '100.64.0.0/10' -# - '169.254.0.0/16' -# -# List of IP address CIDR ranges that the URL preview spider is allowed -# to access even if they are specified in url_preview_ip_range_blacklist. -# This is useful for specifying exceptions to wide-ranging blacklisted -# target IP ranges - e.g. for enabling URL previews for a specific private -# website only visible in your network. -# -# url_preview_ip_range_whitelist: -# - '192.168.1.1' - -# Optional list of URL matches that the URL preview spider is -# denied from accessing. You should use url_preview_ip_range_blacklist -# in preference to this, otherwise someone could define a public DNS -# entry that points to a private IP address and circumvent the blacklist. -# This is more useful if you know there is an entire shape of URL that -# you know that will never want synapse to try to spider. -# -# Each list entry is a dictionary of url component attributes as returned -# by urlparse.urlsplit as applied to the absolute form of the URL. See -# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit -# The values of the dictionary are treated as an filename match pattern -# applied to that component of URLs, unless they start with a ^ in which -# case they are treated as a regular expression match. If all the -# specified component matches for a given list item succeed, the URL is -# blacklisted. -# -# url_preview_url_blacklist: -# # blacklist any URL with a username in its URI -# - username: '*' -# -# # blacklist all *.google.com URLs -# - netloc: 'google.com' -# - netloc: '*.google.com' -# -# # blacklist all plain HTTP URLs -# - scheme: 'http' -# -# # blacklist http(s)://www.acme.com/foo -# - netloc: 'www.acme.com' -# path: '/foo' -# -# # blacklist any URL with a literal IPv4 address -# - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' - -# The largest allowed URL preview spidering size in bytes -max_spider_size: "10M" - - - - -## Captcha ## -# See docs/CAPTCHA_SETUP for full details of configuring this. - -# This Home Server's ReCAPTCHA public key. -recaptcha_public_key: "YOUR_PUBLIC_KEY" - -# This Home Server's ReCAPTCHA private key. -recaptcha_private_key: "YOUR_PRIVATE_KEY" - -# Enables ReCaptcha checks when registering, preventing signup -# unless a captcha is answered. Requires a valid ReCaptcha -# public/private key. -enable_registration_captcha: False - -# A secret key used to bypass the captcha test entirely. -#captcha_bypass_secret: "YOUR_SECRET_HERE" - -# The API endpoint to use for verifying m.login.recaptcha responses. -recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" - - -## Turn ## - -# The public URIs of the TURN server to give to clients -turn_uris: [] - -# The shared secret used to compute passwords for the TURN server -turn_shared_secret: "YOUR_SHARED_SECRET" - -# The Username and password if the TURN server needs them and -# does not use a token -#turn_username: "TURNSERVER_USERNAME" -#turn_password: "TURNSERVER_PASSWORD" - -# How long generated TURN credentials last -turn_user_lifetime: "1h" - -# Whether guests should be allowed to use the TURN server. -# This defaults to True, otherwise VoIP will be unreliable for guests. -# However, it does introduce a slight security risk as it allows users to -# connect to arbitrary endpoints without having first signed up for a -# valid account (e.g. by passing a CAPTCHA). -turn_allow_guests: False - - -## Registration ## - -# Enable registration for new users. -enable_registration: False - -# The user must provide all of the below types of 3PID when registering. -# -# registrations_require_3pid: -# - email -# - msisdn - -# Mandate that users are only allowed to associate certain formats of -# 3PIDs with accounts on this server. -# -# allowed_local_3pids: -# - medium: email -# pattern: ".*@matrix\.org" -# - medium: email -# pattern: ".*@vector\.im" -# - medium: msisdn -# pattern: "\+44" - -# If set, allows registration by anyone who also has the shared -# secret, even if registration is otherwise disabled. -# registration_shared_secret: <PRIVATE STRING> - -# Set the number of bcrypt rounds used to generate password hash. -# Larger numbers increase the work factor needed to generate the hash. -# The default number is 12 (which equates to 2^12 rounds). -# N.B. that increasing this will exponentially increase the time required -# to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. -bcrypt_rounds: 12 - -# Allows users to register as guests without a password/email/etc, and -# participate in rooms hosted on this server which have been made -# accessible to anonymous users. -allow_guest_access: False - -# The list of identity servers trusted to verify third party -# identifiers by this server. -trusted_third_party_id_servers: - - matrix.org - - vector.im - - riot.im - -# Users who register on this homeserver will automatically be joined -# to these rooms -#auto_join_rooms: -# - "#example:example.com" - - -## Metrics ### - -# Enable collection and rendering of performance metrics -enable_metrics: False - -## API Configuration ## - -# A list of event types that will be included in the room_invite_state -room_invite_state_types: - - "m.room.join_rules" - - "m.room.canonical_alias" - - "m.room.avatar" - - "m.room.name" - - -# A list of application service config file to use -app_service_config_files: [] - - -# macaroon_secret_key: <PRIVATE STRING> - -# Used to enable access token expiration. -expire_access_token: False - -## Signing Keys ## - -# Path to the signing key to sign messages with -signing_key_path: "/etc/matrix-synapse/homeserver.signing.key" - -# The keys that the server used to sign messages with but won't use -# to sign new messages. E.g. it has lost its private key -old_signing_keys: {} -# "ed25519:auto": -# # Base64 encoded public key -# key: "The public part of your old signing key." -# # Millisecond POSIX timestamp when the key expired. -# expired_ts: 123456789123 - -# How long key response published by this server is valid for. -# Used to set the valid_until_ts in /key/v2 APIs. -# Determines how quickly servers will query to check which keys -# are still valid. -key_refresh_interval: "1d" # 1 Day. - -# The trusted servers to download signing keys from. -perspectives: - servers: - "matrix.org": - verify_keys: - "ed25519:auto": - key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" - - - -# Enable SAML2 for registration and login. Uses pysaml2 -# config_path: Path to the sp_conf.py configuration file -# idp_redirect_url: Identity provider URL which will redirect -# the user back to /login/saml2 with proper info. -# See pysaml2 docs for format of config. -#saml2_config: -# enabled: true -# config_path: "/home/erikj/git/synapse/sp_conf.py" -# idp_redirect_url: "http://test/idp" - - - -# Enable CAS for registration and login. -#cas_config: -# enabled: true -# server_url: "https://cas-server.com" -# service_url: "https://homeserver.domain.com:8448" -# #required_attributes: -# # name: value - - -# The JWT needs to contain a globally unique "sub" (subject) claim. -# -# jwt_config: -# enabled: true -# secret: "a secret" -# algorithm: "HS256" - - - -# Enable password for login. -password_config: - enabled: true - # Uncomment and change to a secret random string for extra security. - # DO NOT CHANGE THIS AFTER INITIAL SETUP! - #pepper: "" - - - -# Enable sending emails for notification events -# Defining a custom URL for Riot is only needed if email notifications -# should contain links to a self-hosted installation of Riot; when set -# the "app_name" setting is ignored. -# -# If your SMTP server requires authentication, the optional smtp_user & -# smtp_pass variables should be used -# -#email: -# enable_notifs: false -# smtp_host: "localhost" -# smtp_port: 25 -# smtp_user: "exampleusername" -# smtp_pass: "examplepassword" -# require_transport_security: False -# notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>" -# app_name: Matrix -# template_dir: res/templates -# notif_template_html: notif_mail.html -# notif_template_text: notif_mail.txt -# notif_for_new_users: True -# riot_base_url: "http://localhost/riot" - - -# password_providers: -# - module: "ldap_auth_provider.LdapAuthProvider" -# config: -# enabled: true -# uri: "ldap://ldap.example.com:389" -# start_tls: true -# base: "ou=users,dc=example,dc=com" -# attributes: -# uid: "cn" -# mail: "email" -# name: "givenName" -# #bind_dn: -# #bind_password: -# #filter: "(objectClass=posixAccount)" - - - -# Clients requesting push notifications can either have the body of -# the message sent in the notification poke along with other details -# like the sender, or just the event ID and room ID (`event_id_only`). -# If clients choose the former, this option controls whether the -# notification request includes the content of the event (other details -# like the sender are still included). For `event_id_only` push, it -# has no effect. - -# For modern android devices the notification content will still appear -# because it is loaded by the app. iPhone, however will send a -# notification saying only that a message arrived and who it came from. -# -#push: -# include_content: true - - -# spam_checker: -# module: "my_custom_project.SuperSpamChecker" -# config: -# example_option: 'things' - - -# Whether to allow non server admins to create groups on this server -enable_group_creation: false - -# If enabled, non server admins can only create groups with local parts -# starting with this prefix -# group_creation_prefix: "unofficial/" - - - -# User Directory configuration -# -# 'search_all_users' defines whether to search all users visible to your HS -# when searching the user directory, rather than limiting to users visible -# in public rooms. Defaults to false. If you set it True, you'll have to run -# UPDATE user_directory_stream_pos SET stream_id = NULL; -# on your database to tell it to rebuild the user_directory search indexes. -# -#user_directory: -# search_all_users: false diff --git a/debian/install b/debian/install index d1f42c8be6..3d916a9718 100644 --- a/debian/install +++ b/debian/install @@ -1,2 +1 @@ -debian/homeserver.yaml etc/matrix-synapse debian/log.yaml etc/matrix-synapse diff --git a/synapse/api/auth.py b/synapse/api/auth.py index ba1019b9b2..e37b807c94 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -819,7 +819,9 @@ class Auth(object): elif threepid: # If the user does not exist yet, but is signing up with a # reserved threepid then pass auth check - if is_threepid_reserved(self.hs.config, threepid): + if is_threepid_reserved( + self.hs.config.mau_limits_reserved_threepids, threepid + ): return # Else if there is no room in the MAU bucket, bail current_mau = yield self.store.get_monthly_active_count() diff --git a/synapse/config/server.py b/synapse/config/server.py index fb57791098..22dcc87d8a 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -256,7 +256,11 @@ class ServerConfig(Config): # # web_client_location: "/path/to/web/root" - # The public-facing base URL for the client API (not including _matrix/...) + # The public-facing base URL that clients use to access this HS + # (not including _matrix/...). This is the same URL a user would + # enter into the 'custom HS URL' field on their client. If you + # use synapse with a reverse proxy, this should be the URL to reach + # synapse via the proxy. # public_baseurl: https://example.com:8448/ # Set the soft limit on the number of file descriptors synapse can use @@ -420,19 +424,18 @@ class ServerConfig(Config): " service on the given port.") -def is_threepid_reserved(config, threepid): +def is_threepid_reserved(reserved_threepids, threepid): """Check the threepid against the reserved threepid config Args: - config(ServerConfig) - to access server config attributes + reserved_threepids([dict]) - list of reserved threepids threepid(dict) - The threepid to test for Returns: boolean Is the threepid undertest reserved_user """ - for tp in config.mau_limits_reserved_threepids: - if (threepid['medium'] == tp['medium'] - and threepid['address'] == tp['address']): + for tp in reserved_threepids: + if (threepid['medium'] == tp['medium'] and threepid['address'] == tp['address']): return True return False diff --git a/synapse/http/client.py b/synapse/http/client.py index afcf698b29..47a1f82ff0 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -333,9 +333,10 @@ class SimpleHttpClient(object): "POST", uri, headers=Headers(actual_headers), data=query_bytes ) + body = yield make_deferred_yieldable(readBody(response)) + if 200 <= response.code < 300: - body = yield make_deferred_yieldable(treq.json_content(response)) - defer.returnValue(body) + defer.returnValue(json.loads(body)) else: raise HttpResponseException(response.code, response.phrase, body) diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 64c780a341..0ec28c6696 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -101,7 +101,8 @@ class MatrixFederationAgent(object): if port is not None: target = (host, port) else: - server_list = yield self._srv_resolver.resolve_service(server_name_bytes) + service_name = b"_matrix._tcp.%s" % (server_name_bytes, ) + server_list = yield self._srv_resolver.resolve_service(service_name) if not server_list: target = (host, 8448) logger.debug("No SRV record for %s, using %s", host, target) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 14025cd219..7f812b8209 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -416,8 +416,11 @@ class RegisterRestServlet(RestServlet): ) # Necessary due to auth checks prior to the threepid being # written to the db - if is_threepid_reserved(self.hs.config, threepid): - yield self.store.upsert_monthly_active_user(registered_user_id) + if threepid: + if is_threepid_reserved( + self.hs.config.mau_limits_reserved_threepids, threepid + ): + yield self.store.upsert_monthly_active_user(registered_user_id) # remember that we've now registered that user account, and with # what user ID (since the user may not have specified) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 254fdc04c6..f62f70b9f1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -26,6 +26,7 @@ from prometheus_client import Histogram from twisted.internet import defer from synapse.api.errors import StoreError +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.engines import PostgresEngine from synapse.util.caches.descriptors import Cache from synapse.util.logcontext import LoggingContext, PreserveLoggingContext @@ -198,7 +199,12 @@ class SQLBaseStore(object): if self.database_engine.can_native_upsert: # Check ASAP (and then later, every 1s) to see if we have finished # background updates of tables that aren't safe to update. - self._clock.call_later(0.0, self._check_safe_to_upsert) + self._clock.call_later( + 0.0, + run_as_background_process, + "upsert_safety_check", + self._check_safe_to_upsert + ) @defer.inlineCallbacks def _check_safe_to_upsert(self): @@ -208,7 +214,7 @@ class SQLBaseStore(object): If there are background updates, we will need to wait, as they may be the addition of indexes that set the UNIQUE constraint that we require. - If the background updates have not completed, wait a second and check again. + If the background updates have not completed, wait 15 sec and check again. """ updates = yield self._simple_select_list( "background_updates", @@ -221,11 +227,16 @@ class SQLBaseStore(object): # The User IPs table in schema #53 was missing a unique index, which we # run as a background update. if "user_ips_device_unique_index" not in updates: - self._unsafe_to_upsert_tables.discard("user_id") + self._unsafe_to_upsert_tables.discard("user_ips") # If there's any tables left to check, reschedule to run. if self._unsafe_to_upsert_tables: - self._clock.call_later(1.0, self._check_safe_to_upsert) + self._clock.call_later( + 15.0, + run_as_background_process, + "upsert_safety_check", + self._check_safe_to_upsert + ) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -609,7 +620,7 @@ class SQLBaseStore(object): inserting lock (bool): True to lock the table when doing the upsert. Returns: - Deferred(None or bool): Native upserts always return None. Emulated + None or bool: Native upserts always return None. Emulated upserts return True if a new entry was created, False if an existing one was updated. """ @@ -637,6 +648,18 @@ class SQLBaseStore(object): def _simple_upsert_txn_emulated( self, txn, table, keyvalues, values, insertion_values={}, lock=True ): + """ + Args: + table (str): The table to upsert into + keyvalues (dict): The unique key tables and their new values + values (dict): The nonunique columns and their new values + insertion_values (dict): additional key/values to use only when + inserting + lock (bool): True to lock the table when doing the upsert. + Returns: + bool: Return True if a new entry was created, False if an existing + one was updated. + """ # We need to lock the table :(, unless we're *really* careful if lock: self.database_engine.lock_table(txn, table) diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 7a3881f558..b32d7566a5 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -26,6 +26,7 @@ from twisted.web.http import HTTPChannel from synapse.crypto.context_factory import ClientTLSOptionsFactory from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent +from synapse.http.federation.srv_resolver import Server from synapse.util.logcontext import LoggingContext from tests.server import FakeTransport, ThreadedMemoryReactorClock @@ -105,7 +106,7 @@ class MatrixFederationAgentTests(TestCase): def test_get(self): """ - happy-path test of a GET request + happy-path test of a GET request with an explicit port """ self.reactor.lookups["testserv"] = "1.2.3.4" test_d = self._make_get_request(b"matrix://testserv:8448/foo/bar") @@ -130,10 +131,6 @@ class MatrixFederationAgentTests(TestCase): request = http_server.requests[0] self.assertEqual(request.method, b'GET') self.assertEqual(request.path, b'/foo/bar') - self.assertEqual( - request.requestHeaders.getRawHeaders(b'host'), - [b'testserv:8448'] - ) content = request.content.read() self.assertEqual(content, b'') @@ -177,7 +174,9 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) - self.mock_resolver.resolve_service.assert_called_once() + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.1.2.3.4", + ) # Make sure treq is trying to connect clients = self.reactor.tcpClients @@ -196,11 +195,87 @@ class MatrixFederationAgentTests(TestCase): request = http_server.requests[0] self.assertEqual(request.method, b'GET') self.assertEqual(request.path, b'/foo/bar') - # XXX currently broken - # self.assertEqual( - # request.requestHeaders.getRawHeaders(b'host'), - # [b'1.2.3.4:8448'] - # ) + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + + def test_get_hostname_no_srv(self): + """ + Test the behaviour when the server name has no port, and no SRV record + """ + + self.mock_resolver.resolve_service.side_effect = lambda _: [] + self.reactor.lookups["testserv"] = "1.2.3.4" + + test_d = self._make_get_request(b"matrix://testserv/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.testserv", + ) + + # Make sure treq is trying to connect + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients[0] + self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 8448) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=b'testserv', + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + + def test_get_hostname_srv(self): + """ + Test the behaviour when there is a single SRV record + """ + self.mock_resolver.resolve_service.side_effect = lambda _: [ + Server(host="srvtarget", port=8443) + ] + self.reactor.lookups["srvtarget"] = "1.2.3.4" + + test_d = self._make_get_request(b"matrix://testserv/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.testserv", + ) + + # Make sure treq is trying to connect + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients[0] + self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 8443) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=b'testserv', + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') # finish the request request.finish() diff --git a/tests/utils.py b/tests/utils.py index 08d6faa0a6..df73c539c3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -154,7 +154,9 @@ def default_config(name): config.update_user_directory = False def is_threepid_reserved(threepid): - return ServerConfig.is_threepid_reserved(config, threepid) + return ServerConfig.is_threepid_reserved( + config.mau_limits_reserved_threepids, threepid + ) config.is_threepid_reserved.side_effect = is_threepid_reserved |