diff --git a/.gitignore b/.gitignore
index 37f0028d66..a20f3e615d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,64 +1,36 @@
-*.pyc
-.*.swp
+# filename patterns
*~
-*.lock
-
-.DS_Store
-_trial_temp/
-_trial_temp*/
-logs/
-dbs/
+.*.swp
+.#*
+*.deb
*.egg
-dist/
-docs/build/
*.egg-info
-pip-wheel-metadata/
-
-cmdclient_config.json
-homeserver*.db
-homeserver*.log
-homeserver*.log.*
-homeserver*.pid
-/homeserver*.yaml
-
-*.signing.key
-*.tls.crt
-*.tls.dh
-*.tls.key
-
-.coverage*
-coverage.*
-!.coveragerc
-htmlcov
-
-demo/*/*.db
-demo/*/*.log
-demo/*/*.log.*
-demo/*/*.pid
-demo/media_store.*
-demo/etc
-
-uploads
-cache
-
-.idea/
-media_store/
-
+*.lock
+*.pyc
*.tac
+_trial_temp/
+_trial_temp*/
-build/
-venv/
-venv*/
-*venv/
-
-localhost-800*/
-static/client/register/register_config.js
-.tox
-
-env/
-*.config
+# stuff that is likely to exist when you run a server locally
+/*.signing.key
+/*.tls.crt
+/*.tls.key
+/uploads
+/media_store/
+
+# IDEs
+/.idea/
+/.ropeproject/
+/.vscode/
+
+# build products
+/.coverage*
+!/.coveragerc
+/.tox
+/build/
+/coverage.*
+/dist/
+/docs/build/
+/htmlcov
+/pip-wheel-metadata/
-.vscode/
-.ropeproject/
-*.deb
-/debs
diff --git a/.travis.yml b/.travis.yml
index d88f10324f..5d763123a0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -89,7 +89,7 @@ install:
- psql -At -U postgres -c 'select version();' || true
- pip install tox
-
+
# if we don't have python3.6 in this environment, travis unhelpfully gives us
# a `python3.6` on our path which does nothing but spit out a warning. Tox
# tries to run it (even if we're not running a py36 env), so the build logs
diff --git a/INSTALL.md b/INSTALL.md
index fb6a5e4e99..2993f3a9e2 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -358,26 +358,25 @@ For information on using a reverse proxy, see
[docs/reverse_proxy.rst](docs/reverse_proxy.rst).
To configure Synapse to expose an HTTPS port, you will need to edit
-`homeserver.yaml`.
-
-First, under the `listeners` section, uncomment the configuration for the
-TLS-enabled listener. (Remove the hash sign (`#`) and space at the start of
-each line). The relevant lines are like this:
-
-```
- - port: 8448
- type: http
- tls: true
- resources:
- - names: [client, federation]
-```
-
-You will also need to uncomment the `tls_certificate_path` and
-`tls_private_key_path` lines under the `TLS` section. You can either point
-these settings at an existing certificate and key, or you can enable Synapse's
-built-in ACME (Let's Encrypt) support. Instructions for having Synapse
-automatically provision and renew federation certificates through ACME can be
-found at [ACME.md](docs/ACME.md).
+`homeserver.yaml`, as follows:
+
+* First, under the `listeners` section, uncomment the configuration for the
+ TLS-enabled listener. (Remove the hash sign (`#`) at the start of
+ each line). The relevant lines are like this:
+
+ ```
+ - port: 8448
+ type: http
+ tls: true
+ resources:
+ - names: [client, federation]
+ ```
+* You will also need to uncomment the `tls_certificate_path` and
+ `tls_private_key_path` lines under the `TLS` section. You can either
+ point these settings at an existing certificate and key, or you can
+ enable Synapse's built-in ACME (Let's Encrypt) support. Instructions
+ for having Synapse automatically provision and renew federation
+ certificates through ACME can be found at [ACME.md](docs/ACME.md).
## Registering a user
diff --git a/changelog.d/4644.misc b/changelog.d/4644.misc
new file mode 100644
index 0000000000..84137c3412
--- /dev/null
+++ b/changelog.d/4644.misc
@@ -0,0 +1 @@
+Introduce upsert batching functionality in the database layer.
diff --git a/changelog.d/4681.misc b/changelog.d/4681.misc
new file mode 100644
index 0000000000..37d3588804
--- /dev/null
+++ b/changelog.d/4681.misc
@@ -0,0 +1 @@
+Attempt to clarify installation instructions/config
diff --git a/changelog.d/4688.misc b/changelog.d/4688.misc
new file mode 100644
index 0000000000..24cd2eb424
--- /dev/null
+++ b/changelog.d/4688.misc
@@ -0,0 +1 @@
+Clean up gitignores
diff --git a/changelog.d/4690.bugfix b/changelog.d/4690.bugfix
new file mode 100644
index 0000000000..e4cfc5e413
--- /dev/null
+++ b/changelog.d/4690.bugfix
@@ -0,0 +1 @@
+Fix TaskStopped exceptions in logs when outbound requests time out.
\ No newline at end of file
diff --git a/changelog.d/4691.misc b/changelog.d/4691.misc
new file mode 100644
index 0000000000..8eb825edf0
--- /dev/null
+++ b/changelog.d/4691.misc
@@ -0,0 +1 @@
+Improve the logging in the pusher process.
diff --git a/changelog.d/4695.feature b/changelog.d/4695.feature
new file mode 100644
index 0000000000..3816c9dec8
--- /dev/null
+++ b/changelog.d/4695.feature
@@ -0,0 +1 @@
+Add prometheus metrics for number of outgoing EDUs, by type.
diff --git a/demo/.gitignore b/demo/.gitignore
new file mode 100644
index 0000000000..4d12712343
--- /dev/null
+++ b/demo/.gitignore
@@ -0,0 +1,7 @@
+*.db
+*.log
+*.log.*
+*.pid
+
+/media_store.*
+/etc
diff --git a/synapse/config/api.py b/synapse/config/api.py
index 9f25bbc5cb..e8a753f002 100644
--- a/synapse/config/api.py
+++ b/synapse/config/api.py
@@ -33,6 +33,7 @@ class ApiConfig(Config):
## API Configuration ##
# A list of event types that will be included in the room_invite_state
+ #
room_invite_state_types:
- "{JoinRules}"
- "{CanonicalAlias}"
diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index c21cb3dd87..c260d59464 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -38,10 +38,12 @@ class AppServiceConfig(Config):
def default_config(cls, **kwargs):
return """\
# A list of application service config file to use
+ #
app_service_config_files: []
# Whether or not to track application service IP addresses. Implicitly
# enables MAU tracking for application service users.
+ #
track_appservice_user_ips: False
"""
diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py
index 7ba0c2de6a..4064891ffb 100644
--- a/synapse/config/captcha.py
+++ b/synapse/config/captcha.py
@@ -30,14 +30,17 @@ class CaptchaConfig(Config):
# See docs/CAPTCHA_SETUP for full details of configuring this.
# This Home Server's ReCAPTCHA public key.
+ #
recaptcha_public_key: "YOUR_PUBLIC_KEY"
# This Home Server's ReCAPTCHA private key.
+ #
recaptcha_private_key: "YOUR_PRIVATE_KEY"
# Enables ReCaptcha checks when registering, preventing signup
# unless a captcha is answered. Requires a valid ReCaptcha
# public/private key.
+ #
enable_registration_captcha: False
# A secret key used to bypass the captcha test entirely.
diff --git a/synapse/config/cas.py b/synapse/config/cas.py
index 8109e5f95e..609c0815c8 100644
--- a/synapse/config/cas.py
+++ b/synapse/config/cas.py
@@ -38,6 +38,7 @@ class CasConfig(Config):
def default_config(self, config_dir_path, server_name, **kwargs):
return """
# Enable CAS for registration and login.
+ #
#cas_config:
# enabled: true
# server_url: "https://cas-server.com"
diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 9f2e85342f..abeb0180d3 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -54,20 +54,20 @@ DEFAULT_CONFIG = """\
# for an account. Has no effect unless `require_at_registration` is enabled.
# Defaults to "Privacy Policy".
#
-# user_consent:
-# template_dir: res/templates/privacy
-# version: 1.0
-# server_notice_content:
-# msgtype: m.text
-# body: >-
-# To continue using this homeserver you must review and agree to the
-# terms and conditions at %(consent_uri)s
-# send_server_notice_to_guests: True
-# block_events_error: >-
-# To continue using this homeserver you must review and agree to the
-# terms and conditions at %(consent_uri)s
-# require_at_registration: False
-# policy_name: Privacy Policy
+#user_consent:
+# template_dir: res/templates/privacy
+# version: 1.0
+# server_notice_content:
+# msgtype: m.text
+# body: >-
+# To continue using this homeserver you must review and agree to the
+# terms and conditions at %(consent_uri)s
+# send_server_notice_to_guests: True
+# block_events_error: >-
+# To continue using this homeserver you must review and agree to the
+# terms and conditions at %(consent_uri)s
+# require_at_registration: False
+# policy_name: Privacy Policy
#
"""
diff --git a/synapse/config/groups.py b/synapse/config/groups.py
index 997fa2881f..46933a904c 100644
--- a/synapse/config/groups.py
+++ b/synapse/config/groups.py
@@ -24,9 +24,11 @@ class GroupsConfig(Config):
def default_config(self, **kwargs):
return """\
# Whether to allow non server admins to create groups on this server
+ #
enable_group_creation: false
# If enabled, non server admins can only create groups with local parts
# starting with this prefix
- # group_creation_prefix: "unofficial/"
+ #
+ #group_creation_prefix: "unofficial/"
"""
diff --git a/synapse/config/jwt_config.py b/synapse/config/jwt_config.py
index 51e7f7e003..ecb4124096 100644
--- a/synapse/config/jwt_config.py
+++ b/synapse/config/jwt_config.py
@@ -46,8 +46,8 @@ class JWTConfig(Config):
return """\
# The JWT needs to contain a globally unique "sub" (subject) claim.
#
- # jwt_config:
- # enabled: true
- # secret: "a secret"
- # algorithm: "HS256"
+ #jwt_config:
+ # enabled: true
+ # secret: "a secret"
+ # algorithm: "HS256"
"""
diff --git a/synapse/config/key.py b/synapse/config/key.py
index 499ffd4e06..35f05fa974 100644
--- a/synapse/config/key.py
+++ b/synapse/config/key.py
@@ -40,7 +40,7 @@ class KeyConfig(Config):
def read_config(self, config):
self.signing_key = self.read_signing_key(config["signing_key_path"])
self.old_signing_keys = self.read_old_signing_keys(
- config["old_signing_keys"]
+ config.get("old_signing_keys", {})
)
self.key_refresh_interval = self.parse_duration(
config["key_refresh_interval"]
@@ -83,24 +83,29 @@ class KeyConfig(Config):
# a secret which is used to sign access tokens. If none is specified,
# the registration_shared_secret is used, if one is given; otherwise,
# a secret key is derived from the signing key.
+ #
%(macaroon_secret_key)s
# Used to enable access token expiration.
+ #
expire_access_token: False
# a secret which is used to calculate HMACs for form values, to stop
# falsification of values. Must be specified for the User Consent
# forms to work.
+ #
%(form_secret)s
## Signing Keys ##
# Path to the signing key to sign messages with
+ #
signing_key_path: "%(base_key_name)s.signing.key"
# The keys that the server used to sign messages with but won't use
# to sign new messages. E.g. it has lost its private key
- old_signing_keys: {}
+ #
+ #old_signing_keys:
# "ed25519:auto":
# # Base64 encoded public key
# key: "The public part of your old signing key."
@@ -111,9 +116,11 @@ class KeyConfig(Config):
# Used to set the valid_until_ts in /key/v2 APIs.
# Determines how quickly servers will query to check which keys
# are still valid.
+ #
key_refresh_interval: "1d" # 1 Day.
# The trusted servers to download signing keys from.
+ #
perspectives:
servers:
"matrix.org":
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index 9b5994d55e..f6940b65fd 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -83,6 +83,7 @@ class LoggingConfig(Config):
log_config = os.path.join(config_dir_path, server_name + ".log.config")
return """
# A yaml python logging config file
+ #
log_config: "%(log_config)s"
""" % locals()
diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py
index f7833053ba..ed0498c634 100644
--- a/synapse/config/metrics.py
+++ b/synapse/config/metrics.py
@@ -47,6 +47,7 @@ class MetricsConfig(Config):
## Metrics ###
# Enable collection and rendering of performance metrics
+ #
enable_metrics: False
# Enable sentry integration
@@ -55,6 +56,7 @@ class MetricsConfig(Config):
# this option the sentry server may therefore receive sensitive
# information, and it in turn may then diseminate sensitive information
# through insecure notification channels if so configured.
+ #
#sentry:
# dsn: "..."
diff --git a/synapse/config/password.py b/synapse/config/password.py
index a4bd171399..2a52b9db54 100644
--- a/synapse/config/password.py
+++ b/synapse/config/password.py
@@ -28,6 +28,7 @@ class PasswordConfig(Config):
def default_config(self, config_dir_path, server_name, **kwargs):
return """
# Enable password for login.
+ #
password_config:
enabled: true
# Uncomment and change to a secret random string for extra security.
diff --git a/synapse/config/password_auth_providers.py b/synapse/config/password_auth_providers.py
index f4066abc28..f0a6be0679 100644
--- a/synapse/config/password_auth_providers.py
+++ b/synapse/config/password_auth_providers.py
@@ -52,18 +52,18 @@ class PasswordAuthProviderConfig(Config):
def default_config(self, **kwargs):
return """\
- # password_providers:
- # - module: "ldap_auth_provider.LdapAuthProvider"
- # config:
- # enabled: true
- # uri: "ldap://ldap.example.com:389"
- # start_tls: true
- # base: "ou=users,dc=example,dc=com"
- # attributes:
- # uid: "cn"
- # mail: "email"
- # name: "givenName"
- # #bind_dn:
- # #bind_password:
- # #filter: "(objectClass=posixAccount)"
+ #password_providers:
+ # - module: "ldap_auth_provider.LdapAuthProvider"
+ # config:
+ # enabled: true
+ # uri: "ldap://ldap.example.com:389"
+ # start_tls: true
+ # base: "ou=users,dc=example,dc=com"
+ # attributes:
+ # uid: "cn"
+ # mail: "email"
+ # name: "givenName"
+ # #bind_dn:
+ # #bind_password:
+ # #filter: "(objectClass=posixAccount)"
"""
diff --git a/synapse/config/push.py b/synapse/config/push.py
index b7e0d46afa..62c0060c9c 100644
--- a/synapse/config/push.py
+++ b/synapse/config/push.py
@@ -51,11 +51,11 @@ class PushConfig(Config):
# notification request includes the content of the event (other details
# like the sender are still included). For `event_id_only` push, it
# has no effect.
-
+ #
# For modern android devices the notification content will still appear
# because it is loaded by the app. iPhone, however will send a
# notification saying only that a message arrived and who it came from.
#
#push:
- # include_content: true
+ # include_content: true
"""
diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py
index 83b22dc199..54b71e6841 100644
--- a/synapse/config/ratelimiting.py
+++ b/synapse/config/ratelimiting.py
@@ -32,27 +32,34 @@ class RatelimitConfig(Config):
## Ratelimiting ##
# Number of messages a client can send per second
+ #
rc_messages_per_second: 0.2
# Number of message a client can send before being throttled
+ #
rc_message_burst_count: 10.0
# The federation window size in milliseconds
+ #
federation_rc_window_size: 1000
# The number of federation requests from a single server in a window
# before the server will delay processing the request.
+ #
federation_rc_sleep_limit: 10
# The duration in milliseconds to delay processing events from
# remote servers by if they go over the sleep limit.
+ #
federation_rc_sleep_delay: 500
# The maximum number of concurrent federation requests allowed
# from a single server
+ #
federation_rc_reject_limit: 50
# The number of federation requests to concurrently process from a
# single server
+ #
federation_rc_concurrent: 3
"""
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index d808a989f3..2881482f96 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -70,28 +70,29 @@ class RegistrationConfig(Config):
# The user must provide all of the below types of 3PID when registering.
#
- # registrations_require_3pid:
- # - email
- # - msisdn
+ #registrations_require_3pid:
+ # - email
+ # - msisdn
# Explicitly disable asking for MSISDNs from the registration
# flow (overrides registrations_require_3pid if MSISDNs are set as required)
#
- # disable_msisdn_registration = True
+ #disable_msisdn_registration: True
# Mandate that users are only allowed to associate certain formats of
# 3PIDs with accounts on this server.
#
- # allowed_local_3pids:
- # - medium: email
- # pattern: '.*@matrix\\.org'
- # - medium: email
- # pattern: '.*@vector\\.im'
- # - medium: msisdn
- # pattern: '\\+44'
+ #allowed_local_3pids:
+ # - medium: email
+ # pattern: '.*@matrix\\.org'
+ # - medium: email
+ # pattern: '.*@vector\\.im'
+ # - medium: msisdn
+ # pattern: '\\+44'
# If set, allows registration by anyone who also has the shared
# secret, even if registration is otherwise disabled.
+ #
%(registration_shared_secret)s
# Set the number of bcrypt rounds used to generate password hash.
@@ -99,11 +100,13 @@ class RegistrationConfig(Config):
# The default number is 12 (which equates to 2^12 rounds).
# N.B. that increasing this will exponentially increase the time required
# to register or login - e.g. 24 => 2^24 rounds which will take >20 mins.
+ #
bcrypt_rounds: 12
# Allows users to register as guests without a password/email/etc, and
# participate in rooms hosted on this server which have been made
# accessible to anonymous users.
+ #
allow_guest_access: False
# The identity server which we suggest that clients should use when users log
@@ -112,27 +115,30 @@ class RegistrationConfig(Config):
# (By default, no suggestion is made, so it is left up to the client.
# This setting is ignored unless public_baseurl is also set.)
#
- # default_identity_server: https://matrix.org
+ #default_identity_server: https://matrix.org
# The list of identity servers trusted to verify third party
# identifiers by this server.
#
# Also defines the ID server which will be called when an account is
# deactivated (one will be picked arbitrarily).
+ #
trusted_third_party_id_servers:
- - matrix.org
- - vector.im
+ - matrix.org
+ - vector.im
# Users who register on this homeserver will automatically be joined
# to these rooms
+ #
#auto_join_rooms:
- # - "#example:example.com"
+ # - "#example:example.com"
# Where auto_join_rooms are specified, setting this flag ensures that the
# the rooms exist by creating them when the first user on the
# homeserver registers.
# Setting to false means that if the rooms are not manually created,
# users cannot be auto-joined since they do not exist.
+ #
autocreate_auto_join_rooms: true
""" % locals()
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index 76e3340a91..97db2a5b7a 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -180,29 +180,34 @@ class ContentRepositoryConfig(Config):
uploads_path = os.path.join(data_dir_path, "uploads")
return r"""
# Directory where uploaded images and attachments are stored.
+ #
media_store_path: "%(media_store)s"
# Media storage providers allow media to be stored in different
# locations.
- # media_storage_providers:
- # - module: file_system
- # # Whether to write new local files.
- # store_local: false
- # # Whether to write new remote media
- # store_remote: false
- # # Whether to block upload requests waiting for write to this
- # # provider to complete
- # store_synchronous: false
- # config:
- # directory: /mnt/some/other/directory
+ #
+ #media_storage_providers:
+ # - module: file_system
+ # # Whether to write new local files.
+ # store_local: false
+ # # Whether to write new remote media
+ # store_remote: false
+ # # Whether to block upload requests waiting for write to this
+ # # provider to complete
+ # store_synchronous: false
+ # config:
+ # directory: /mnt/some/other/directory
# Directory where in-progress uploads are stored.
+ #
uploads_path: "%(uploads_path)s"
# The largest allowed upload size in bytes
+ #
max_upload_size: "10M"
# Maximum number of pixels that will be thumbnailed
+ #
max_image_pixels: "32M"
# Whether to generate new thumbnails on the fly to precisely match
@@ -210,9 +215,11 @@ class ContentRepositoryConfig(Config):
# a new resolution is requested by the client the server will
# generate a new thumbnail. If false the server will pick a thumbnail
# from a precalculated list.
+ #
dynamic_thumbnails: false
- # List of thumbnail to precalculate when an image is uploaded.
+ # List of thumbnails to precalculate when an image is uploaded.
+ #
thumbnail_sizes:
- width: 32
height: 32
@@ -233,6 +240,7 @@ class ContentRepositoryConfig(Config):
# Is the preview URL API enabled? If enabled, you *must* specify
# an explicit url_preview_ip_range_blacklist of IPs that the spider is
# denied from accessing.
+ #
url_preview_enabled: False
# List of IP address CIDR ranges that the URL preview spider is denied
@@ -243,16 +251,16 @@ class ContentRepositoryConfig(Config):
# synapse to issue arbitrary GET requests to your internal services,
# causing serious security issues.
#
- # url_preview_ip_range_blacklist:
- # - '127.0.0.0/8'
- # - '10.0.0.0/8'
- # - '172.16.0.0/12'
- # - '192.168.0.0/16'
- # - '100.64.0.0/10'
- # - '169.254.0.0/16'
- # - '::1/128'
- # - 'fe80::/64'
- # - 'fc00::/7'
+ #url_preview_ip_range_blacklist:
+ # - '127.0.0.0/8'
+ # - '10.0.0.0/8'
+ # - '172.16.0.0/12'
+ # - '192.168.0.0/16'
+ # - '100.64.0.0/10'
+ # - '169.254.0.0/16'
+ # - '::1/128'
+ # - 'fe80::/64'
+ # - 'fc00::/7'
#
# List of IP address CIDR ranges that the URL preview spider is allowed
# to access even if they are specified in url_preview_ip_range_blacklist.
@@ -260,8 +268,8 @@ class ContentRepositoryConfig(Config):
# target IP ranges - e.g. for enabling URL previews for a specific private
# website only visible in your network.
#
- # url_preview_ip_range_whitelist:
- # - '192.168.1.1'
+ #url_preview_ip_range_whitelist:
+ # - '192.168.1.1'
# Optional list of URL matches that the URL preview spider is
# denied from accessing. You should use url_preview_ip_range_blacklist
@@ -279,26 +287,25 @@ class ContentRepositoryConfig(Config):
# specified component matches for a given list item succeed, the URL is
# blacklisted.
#
- # url_preview_url_blacklist:
- # # blacklist any URL with a username in its URI
- # - username: '*'
+ #url_preview_url_blacklist:
+ # # blacklist any URL with a username in its URI
+ # - username: '*'
#
- # # blacklist all *.google.com URLs
- # - netloc: 'google.com'
- # - netloc: '*.google.com'
+ # # blacklist all *.google.com URLs
+ # - netloc: 'google.com'
+ # - netloc: '*.google.com'
#
- # # blacklist all plain HTTP URLs
- # - scheme: 'http'
+ # # blacklist all plain HTTP URLs
+ # - scheme: 'http'
#
- # # blacklist http(s)://www.acme.com/foo
- # - netloc: 'www.acme.com'
- # path: '/foo'
+ # # blacklist http(s)://www.acme.com/foo
+ # - netloc: 'www.acme.com'
+ # path: '/foo'
#
- # # blacklist any URL with a literal IPv4 address
- # - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
+ # # blacklist any URL with a literal IPv4 address
+ # - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
# The largest allowed URL preview spidering size in bytes
max_spider_size: "10M"
-
""" % locals()
diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py
index c8e0abbae7..9b897abe3c 100644
--- a/synapse/config/room_directory.py
+++ b/synapse/config/room_directory.py
@@ -76,11 +76,11 @@ class RoomDirectoryConfig(Config):
#
# The default is:
#
- # alias_creation_rules:
- # - user_id: "*"
- # alias: "*"
- # room_id: "*"
- # action: allow
+ #alias_creation_rules:
+ # - user_id: "*"
+ # alias: "*"
+ # room_id: "*"
+ # action: allow
# The `room_list_publication_rules` option controls who can publish and
# which rooms can be published in the public room list.
@@ -105,11 +105,11 @@ class RoomDirectoryConfig(Config):
#
# The default is:
#
- # room_list_publication_rules:
- # - user_id: "*"
- # alias: "*"
- # room_id: "*"
- # action: allow
+ #room_list_publication_rules:
+ # - user_id: "*"
+ # alias: "*"
+ # room_id: "*"
+ # action: allow
"""
def is_alias_creation_allowed(self, user_id, room_id, alias):
diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py
index 86ffe334f5..aff0a1f00c 100644
--- a/synapse/config/saml2_config.py
+++ b/synapse/config/saml2_config.py
@@ -67,44 +67,43 @@ class SAML2Config(Config):
return """
# Enable SAML2 for registration and login. Uses pysaml2.
#
- # saml2_config:
+ # `sp_config` is the configuration for the pysaml2 Service Provider.
+ # See pysaml2 docs for format of config.
#
- # # The following is the configuration for the pysaml2 Service Provider.
- # # See pysaml2 docs for format of config.
- # #
- # # Default values will be used for the 'entityid' and 'service' settings,
- # # so it is not normally necessary to specify them unless you need to
- # # override them.
+ # Default values will be used for the 'entityid' and 'service' settings,
+ # so it is not normally necessary to specify them unless you need to
+ # override them.
#
- # sp_config:
- # # point this to the IdP's metadata. You can use either a local file or
- # # (preferably) a URL.
- # metadata:
- # # local: ["saml2/idp.xml"]
- # remote:
- # - url: https://our_idp/metadata.xml
+ #saml2_config:
+ # sp_config:
+ # # point this to the IdP's metadata. You can use either a local file or
+ # # (preferably) a URL.
+ # metadata:
+ # #local: ["saml2/idp.xml"]
+ # remote:
+ # - url: https://our_idp/metadata.xml
#
- # # The following is just used to generate our metadata xml, and you
- # # may well not need it, depending on your setup. Alternatively you
- # # may need a whole lot more detail - see the pysaml2 docs!
+ # # The rest of sp_config is just used to generate our metadata xml, and you
+ # # may well not need it, depending on your setup. Alternatively you
+ # # may need a whole lot more detail - see the pysaml2 docs!
#
- # description: ["My awesome SP", "en"]
- # name: ["Test SP", "en"]
+ # description: ["My awesome SP", "en"]
+ # name: ["Test SP", "en"]
#
- # organization:
- # name: Example com
- # display_name:
- # - ["Example co", "en"]
- # url: "http://example.com"
+ # organization:
+ # name: Example com
+ # display_name:
+ # - ["Example co", "en"]
+ # url: "http://example.com"
#
- # contact_person:
- # - given_name: Bob
- # sur_name: "the Sysadmin"
- # email_address": ["admin@example.com"]
- # contact_type": technical
+ # contact_person:
+ # - given_name: Bob
+ # sur_name: "the Sysadmin"
+ # email_address": ["admin@example.com"]
+ # contact_type": technical
#
- # # Instead of putting the config inline as above, you can specify a
- # # separate pysaml2 configuration file:
- # #
- # # config_path: "%(config_dir_path)s/sp_conf.py"
+ # # Instead of putting the config inline as above, you can specify a
+ # # separate pysaml2 configuration file:
+ # #
+ # config_path: "%(config_dir_path)s/sp_conf.py"
""" % {"config_dir_path": config_dir_path}
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 93a30e4cfa..4200f10da3 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -286,19 +286,20 @@ class ServerConfig(Config):
#
# This setting requires the affinity package to be installed!
#
- # cpu_affinity: 0xFFFFFFFF
+ #cpu_affinity: 0xFFFFFFFF
# The path to the web client which will be served at /_matrix/client/
# if 'webclient' is configured under the 'listeners' configuration.
#
- # web_client_location: "/path/to/web/root"
+ #web_client_location: "/path/to/web/root"
# The public-facing base URL that clients use to access this HS
# (not including _matrix/...). This is the same URL a user would
# enter into the 'custom HS URL' field on their client. If you
# use synapse with a reverse proxy, this should be the URL to reach
# synapse via the proxy.
- # public_baseurl: https://example.com/
+ #
+ #public_baseurl: https://example.com/
# Set the soft limit on the number of file descriptors synapse can use
# Zero is used to indicate synapse should set the soft limit to the
@@ -309,15 +310,25 @@ class ServerConfig(Config):
use_presence: true
# The GC threshold parameters to pass to `gc.set_threshold`, if defined
- # gc_thresholds: [700, 10, 10]
+ #
+ #gc_thresholds: [700, 10, 10]
# Set the limit on the returned events in the timeline in the get
# and sync operations. The default value is -1, means no upper limit.
- # filter_timeline_limit: 5000
+ #
+ #filter_timeline_limit: 5000
# Whether room invites to users on this server should be blocked
# (except those sent by local server admins). The default is False.
- # block_non_admin_invites: True
+ #
+ #block_non_admin_invites: True
+
+ # Room searching
+ #
+ # If disabled, new messages will not be indexed for searching and users
+ # will receive errors when searching for messages. Defaults to enabled.
+ #
+ #enable_search: false
# Restrict federation to the following whitelist of domains.
# N.B. we recommend also firewalling your federation listener to limit
@@ -325,7 +336,7 @@ class ServerConfig(Config):
# purely on this application-layer restriction. If not specified, the
# default is to whitelist everything.
#
- # federation_domain_whitelist:
+ #federation_domain_whitelist:
# - lon.example.com
# - nyc.example.com
# - syd.example.com
@@ -397,11 +408,11 @@ class ServerConfig(Config):
# will also need to give Synapse a TLS key and certificate: see the TLS section
# below.)
#
- # - port: %(bind_port)s
- # type: http
- # tls: true
- # resources:
- # - names: [client, federation]
+ #- port: %(bind_port)s
+ # type: http
+ # tls: true
+ # resources:
+ # - names: [client, federation]
# Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy
# that unwraps TLS.
@@ -421,52 +432,49 @@ class ServerConfig(Config):
# example additonal_resources:
#
- # additional_resources:
- # "/_matrix/my/custom/endpoint":
- # module: my_module.CustomRequestHandler
- # config: {}
+ #additional_resources:
+ # "/_matrix/my/custom/endpoint":
+ # module: my_module.CustomRequestHandler
+ # config: {}
# Turn on the twisted ssh manhole service on localhost on the given
# port.
- # - port: 9000
- # bind_addresses: ['::1', '127.0.0.1']
- # type: manhole
+ #
+ #- port: 9000
+ # bind_addresses: ['::1', '127.0.0.1']
+ # type: manhole
+
+
+ ## Homeserver blocking ##
- # Homeserver blocking
- #
# How to reach the server admin, used in ResourceLimitError
- # admin_contact: 'mailto:admin@server.com'
- #
- # Global block config
#
- # hs_disabled: False
- # hs_disabled_message: 'Human readable reason for why the HS is blocked'
- # hs_disabled_limit_type: 'error code(str), to help clients decode reason'
+ #admin_contact: 'mailto:admin@server.com'
+
+ # Global blocking
#
+ #hs_disabled: False
+ #hs_disabled_message: 'Human readable reason for why the HS is blocked'
+ #hs_disabled_limit_type: 'error code(str), to help clients decode reason'
+
# Monthly Active User Blocking
#
- # Enables monthly active user checking
- # limit_usage_by_mau: False
- # max_mau_value: 50
- # mau_trial_days: 2
- #
+ #limit_usage_by_mau: False
+ #max_mau_value: 50
+ #mau_trial_days: 2
+
# If enabled, the metrics for the number of monthly active users will
# be populated, however no one will be limited. If limit_usage_by_mau
# is true, this is implied to be true.
- # mau_stats_only: False
#
+ #mau_stats_only: False
+
# Sometimes the server admin will want to ensure certain accounts are
# never blocked by mau checking. These accounts are specified here.
#
- # mau_limit_reserved_threepids:
- # - medium: 'email'
- # address: 'reserved_user@example.com'
- #
- # Room searching
- #
- # If disabled, new messages will not be indexed for searching and users
- # will receive errors when searching for messages. Defaults to enabled.
- # enable_search: true
+ #mau_limit_reserved_threepids:
+ # - medium: 'email'
+ # address: 'reserved_user@example.com'
""" % locals()
def read_arguments(self, args):
diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py
index 3c39850ac6..529dc0a617 100644
--- a/synapse/config/server_notices_config.py
+++ b/synapse/config/server_notices_config.py
@@ -30,11 +30,11 @@ DEFAULT_CONFIG = """\
# It's also possible to override the room name, the display name of the
# "notices" user, and the avatar for the user.
#
-# server_notices:
-# system_mxid_localpart: notices
-# system_mxid_display_name: "Server Notices"
-# system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
-# room_name: "Server Notices"
+#server_notices:
+# system_mxid_localpart: notices
+# system_mxid_display_name: "Server Notices"
+# system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
+# room_name: "Server Notices"
"""
diff --git a/synapse/config/spam_checker.py b/synapse/config/spam_checker.py
index 3fec42bdb0..1502e9faba 100644
--- a/synapse/config/spam_checker.py
+++ b/synapse/config/spam_checker.py
@@ -28,8 +28,8 @@ class SpamCheckerConfig(Config):
def default_config(self, **kwargs):
return """\
- # spam_checker:
- # module: "my_custom_project.SuperSpamChecker"
- # config:
- # example_option: 'things'
+ #spam_checker:
+ # module: "my_custom_project.SuperSpamChecker"
+ # config:
+ # example_option: 'things'
"""
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 38425bb056..8d5d287357 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -177,10 +177,11 @@ class TlsConfig(Config):
# See 'ACME support' below to enable auto-provisioning this certificate via
# Let's Encrypt.
#
- # tls_certificate_path: "%(tls_certificate_path)s"
+ #tls_certificate_path: "%(tls_certificate_path)s"
# PEM-encoded private key for TLS
- # tls_private_key_path: "%(tls_private_key_path)s"
+ #
+ #tls_private_key_path: "%(tls_private_key_path)s"
# ACME support: This will configure Synapse to request a valid TLS certificate
# for your configured `server_name` via Let's Encrypt.
@@ -207,28 +208,28 @@ class TlsConfig(Config):
# ACME support is disabled by default. Uncomment the following line
# (and tls_certificate_path and tls_private_key_path above) to enable it.
#
- # enabled: true
+ #enabled: true
# Endpoint to use to request certificates. If you only want to test,
# use Let's Encrypt's staging url:
# https://acme-staging.api.letsencrypt.org/directory
#
- # url: https://acme-v01.api.letsencrypt.org/directory
+ #url: https://acme-v01.api.letsencrypt.org/directory
# Port number to listen on for the HTTP-01 challenge. Change this if
# you are forwarding connections through Apache/Nginx/etc.
#
- # port: 80
+ #port: 80
# Local addresses to listen on for incoming connections.
# Again, you may want to change this if you are forwarding connections
# through Apache/Nginx/etc.
#
- # bind_addresses: ['::', '0.0.0.0']
+ #bind_addresses: ['::', '0.0.0.0']
# How many days remaining on a certificate before it is renewed.
#
- # reprovision_threshold: 30
+ #reprovision_threshold: 30
# The domain that the certificate should be for. Normally this
# should be the same as your Matrix domain (i.e., 'server_name'), but,
@@ -242,7 +243,7 @@ class TlsConfig(Config):
#
# If not set, defaults to your 'server_name'.
#
- # domain: matrix.example.com
+ #domain: matrix.example.com
# List of allowed TLS fingerprints for this server to publish along
# with the signing keys for this server. Other matrix servers that
@@ -269,8 +270,7 @@ class TlsConfig(Config):
# openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '='
# or by checking matrix.org/federationtester/api/report?server_name=$host
#
- tls_fingerprints: []
- # tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+ #tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
"""
% locals()
diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py
index 38e8947843..fab3a7d1c8 100644
--- a/synapse/config/user_directory.py
+++ b/synapse/config/user_directory.py
@@ -40,5 +40,5 @@ class UserDirectoryConfig(Config):
# on your database to tell it to rebuild the user_directory search indexes.
#
#user_directory:
- # search_all_users: false
+ # search_all_users: false
"""
diff --git a/synapse/config/voip.py b/synapse/config/voip.py
index d07bd24ffd..257f7c86e7 100644
--- a/synapse/config/voip.py
+++ b/synapse/config/voip.py
@@ -27,20 +27,24 @@ class VoipConfig(Config):
def default_config(self, **kwargs):
return """\
- ## Turn ##
+ ## TURN ##
# The public URIs of the TURN server to give to clients
+ #
#turn_uris: []
# The shared secret used to compute passwords for the TURN server
+ #
#turn_shared_secret: "YOUR_SHARED_SECRET"
# The Username and password if the TURN server needs them and
# does not use a token
+ #
#turn_username: "TURNSERVER_USERNAME"
#turn_password: "TURNSERVER_PASSWORD"
# How long generated TURN credentials last
+ #
turn_user_lifetime: "1h"
# Whether guests should be allowed to use the TURN server.
@@ -48,5 +52,6 @@ class VoipConfig(Config):
# However, it does introduce a slight security risk as it allows users to
# connect to arbitrary endpoints without having first signed up for a
# valid account (e.g. by passing a CAPTCHA).
+ #
turn_allow_guests: True
"""
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 1f0b67f5f8..30941f5ad6 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -33,7 +33,6 @@ from synapse.metrics import (
event_processing_loop_counter,
event_processing_loop_room_count,
events_processed_counter,
- sent_edus_counter,
sent_transactions_counter,
)
from synapse.metrics.background_process_metrics import run_as_background_process
@@ -47,10 +46,24 @@ from .units import Edu, Transaction
logger = logging.getLogger(__name__)
sent_pdus_destination_dist_count = Counter(
- "synapse_federation_client_sent_pdu_destinations:count", ""
+ "synapse_federation_client_sent_pdu_destinations:count",
+ "Number of PDUs queued for sending to one or more destinations",
)
+
sent_pdus_destination_dist_total = Counter(
"synapse_federation_client_sent_pdu_destinations:total", ""
+ "Total number of PDUs queued for sending across all destinations",
+)
+
+sent_edus_counter = Counter(
+ "synapse_federation_client_sent_edus",
+ "Total number of EDUs successfully sent",
+)
+
+sent_edus_by_type = Counter(
+ "synapse_federation_client_sent_edus_by_type",
+ "Number of sent EDUs successfully sent, by event type",
+ ["type"],
)
@@ -360,8 +373,6 @@ class TransactionQueue(object):
logger.info("Not sending EDU to ourselves")
return
- sent_edus_counter.inc()
-
if key:
self.pending_edus_keyed_by_dest.setdefault(
destination, {}
@@ -496,6 +507,9 @@ class TransactionQueue(object):
)
if success:
sent_transactions_counter.inc()
+ sent_edus_counter.inc(len(pending_edus))
+ for edu in pending_edus:
+ sent_edus_by_type.labels(edu.edu_type).inc()
# Remove the acknowledged device messages from the database
# Only bother if we actually sent some device messages
if device_message_edus:
diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py
index a3f9e4f67c..d36bcd6336 100644
--- a/synapse/http/__init__.py
+++ b/synapse/http/__init__.py
@@ -15,8 +15,10 @@
# limitations under the License.
import re
+from twisted.internet import task
from twisted.internet.defer import CancelledError
from twisted.python import failure
+from twisted.web.client import FileBodyProducer
from synapse.api.errors import SynapseError
@@ -47,3 +49,16 @@ def redact_uri(uri):
r'\1<redacted>\3',
uri
)
+
+
+class QuieterFileBodyProducer(FileBodyProducer):
+ """Wrapper for FileBodyProducer that avoids CRITICAL errors when the connection drops.
+
+ Workaround for https://github.com/matrix-org/synapse/issues/4003 /
+ https://twistedmatrix.com/trac/ticket/6528
+ """
+ def stopProducing(self):
+ try:
+ FileBodyProducer.stopProducing(self)
+ except task.TaskStopped:
+ pass
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 47a1f82ff0..ad454f4964 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -15,6 +15,7 @@
# limitations under the License.
import logging
+from io import BytesIO
from six import text_type
from six.moves import urllib
@@ -39,7 +40,11 @@ from twisted.web.http import PotentialDataLoss
from twisted.web.http_headers import Headers
from synapse.api.errors import Codes, HttpResponseException, SynapseError
-from synapse.http import cancelled_to_request_timed_out_error, redact_uri
+from synapse.http import (
+ QuieterFileBodyProducer,
+ cancelled_to_request_timed_out_error,
+ redact_uri,
+)
from synapse.util.async_helpers import timeout_deferred
from synapse.util.caches import CACHE_SIZE_FACTOR
from synapse.util.logcontext import make_deferred_yieldable
@@ -246,7 +251,7 @@ class SimpleHttpClient(object):
)
@defer.inlineCallbacks
- def request(self, method, uri, data=b'', headers=None):
+ def request(self, method, uri, data=None, headers=None):
"""
Args:
method (str): HTTP method to use.
@@ -265,11 +270,15 @@ class SimpleHttpClient(object):
logger.info("Sending request %s %s", method, redact_uri(uri))
try:
+ body_producer = None
+ if data is not None:
+ body_producer = QuieterFileBodyProducer(BytesIO(data))
+
request_deferred = treq.request(
method,
uri,
agent=self.agent,
- data=data,
+ data=body_producer,
headers=headers,
**self._extra_treq_args
)
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 3c24bf3805..1682c9af13 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -28,11 +28,10 @@ from canonicaljson import encode_canonical_json
from prometheus_client import Counter
from signedjson.sign import sign_json
-from twisted.internet import defer, protocol, task
+from twisted.internet import defer, protocol
from twisted.internet.error import DNSLookupError
from twisted.internet.task import _EPSILON, Cooperator
from twisted.web._newclient import ResponseDone
-from twisted.web.client import FileBodyProducer
from twisted.web.http_headers import Headers
import synapse.metrics
@@ -44,6 +43,7 @@ from synapse.api.errors import (
RequestSendFailed,
SynapseError,
)
+from synapse.http import QuieterFileBodyProducer
from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent
from synapse.util.async_helpers import timeout_deferred
from synapse.util.logcontext import make_deferred_yieldable
@@ -839,16 +839,3 @@ def encode_query_args(args):
query_bytes = urllib.parse.urlencode(encoded_args, True)
return query_bytes.encode('utf8')
-
-
-class QuieterFileBodyProducer(FileBodyProducer):
- """Wrapper for FileBodyProducer that avoids CRITICAL errors when the connection drops.
-
- Workaround for https://github.com/matrix-org/synapse/issues/4003 /
- https://twistedmatrix.com/trac/ticket/6528
- """
- def stopProducing(self):
- try:
- FileBodyProducer.stopProducing(self)
- except task.TaskStopped:
- pass
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 59900aa5d1..ef48984fdd 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -274,8 +274,6 @@ pending_calls_metric = Histogram(
# Federation Metrics
#
-sent_edus_counter = Counter("synapse_federation_client_sent_edus", "")
-
sent_transactions_counter = Counter("synapse_federation_client_sent_transactions", "")
events_processed_counter = Counter("synapse_federation_client_events_processed", "")
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index 87fa7f006a..98d8d9560b 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -333,10 +333,10 @@ class HttpPusher(object):
defer.returnValue([])
try:
resp = yield self.http_client.post_json_get_json(self.url, notification_dict)
- except Exception:
- logger.warn(
- "Failed to push event %s to %s",
- event.event_id, self.name, exc_info=True,
+ except Exception as e:
+ logger.warning(
+ "Failed to push event %s to %s: %s %s",
+ event.event_id, self.name, type(e), e,
)
defer.returnValue(False)
rejected = []
@@ -367,10 +367,10 @@ class HttpPusher(object):
}
try:
resp = yield self.http_client.post_json_get_json(self.url, d)
- except Exception:
- logger.warn(
- "Failed to send badge count to %s",
- self.name, exc_info=True,
+ except Exception as e:
+ logger.warning(
+ "Failed to send badge count to %s: %s %s",
+ self.name, type(e), e,
)
defer.returnValue(False)
rejected = []
diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py
index fcee6d9d7e..368d5094be 100644
--- a/synapse/push/pusher.py
+++ b/synapse/push/pusher.py
@@ -52,11 +52,12 @@ class PusherFactory(object):
logger.info("defined email pusher type")
def create_pusher(self, pusherdict):
- logger.info("trying to create_pusher for %r", pusherdict)
-
- if pusherdict['kind'] in self.pusher_types:
- logger.info("found pusher")
- return self.pusher_types[pusherdict['kind']](self.hs, pusherdict)
+ kind = pusherdict['kind']
+ f = self.pusher_types.get(kind, None)
+ if not f:
+ return None
+ logger.info("creating %s pusher for %r", kind, pusherdict)
+ return f(self.hs, pusherdict)
def _create_email_pusher(self, _hs, pusherdict):
app_name = self._app_name_from_pusherdict(pusherdict)
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index f1a5366b95..3d895da43c 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -106,6 +106,14 @@ class LoggingTransaction(object):
def __iter__(self):
return self.txn.__iter__()
+ def execute_batch(self, sql, args):
+ if isinstance(self.database_engine, PostgresEngine):
+ from psycopg2.extras import execute_batch
+ self._do_execute(lambda *x: execute_batch(self.txn, *x), sql, args)
+ else:
+ for val in args:
+ self.execute(sql, val)
+
def execute(self, sql, *args):
self._do_execute(self.txn.execute, sql, *args)
@@ -699,20 +707,34 @@ class SQLBaseStore(object):
else:
return "%s = ?" % (key,)
- # First try to update.
- sql = "UPDATE %s SET %s WHERE %s" % (
- table,
- ", ".join("%s = ?" % (k,) for k in values),
- " AND ".join(_getwhere(k) for k in keyvalues)
- )
- sqlargs = list(values.values()) + list(keyvalues.values())
+ if not values:
+ # If `values` is empty, then all of the values we care about are in
+ # the unique key, so there is nothing to UPDATE. We can just do a
+ # SELECT instead to see if it exists.
+ sql = "SELECT 1 FROM %s WHERE %s" % (
+ table,
+ " AND ".join(_getwhere(k) for k in keyvalues)
+ )
+ sqlargs = list(keyvalues.values())
+ txn.execute(sql, sqlargs)
+ if txn.fetchall():
+ # We have an existing record.
+ return False
+ else:
+ # First try to update.
+ sql = "UPDATE %s SET %s WHERE %s" % (
+ table,
+ ", ".join("%s = ?" % (k,) for k in values),
+ " AND ".join(_getwhere(k) for k in keyvalues)
+ )
+ sqlargs = list(values.values()) + list(keyvalues.values())
- txn.execute(sql, sqlargs)
- if txn.rowcount > 0:
- # successfully updated at least one row.
- return False
+ txn.execute(sql, sqlargs)
+ if txn.rowcount > 0:
+ # successfully updated at least one row.
+ return False
- # We didn't update any rows so insert a new one
+ # We didn't find any existing rows, so insert a new one
allvalues = {}
allvalues.update(keyvalues)
allvalues.update(values)
@@ -759,6 +781,106 @@ class SQLBaseStore(object):
)
txn.execute(sql, list(allvalues.values()))
+ def _simple_upsert_many_txn(
+ self, txn, table, key_names, key_values, value_names, value_values
+ ):
+ """
+ Upsert, many times.
+
+ Args:
+ table (str): The table to upsert into
+ key_names (list[str]): The key column names.
+ key_values (list[list]): A list of each row's key column values.
+ value_names (list[str]): The value column names. If empty, no
+ values will be used, even if value_values is provided.
+ value_values (list[list]): A list of each row's value column values.
+ Returns:
+ None
+ """
+ if (
+ self.database_engine.can_native_upsert
+ and table not in self._unsafe_to_upsert_tables
+ ):
+ return self._simple_upsert_many_txn_native_upsert(
+ txn, table, key_names, key_values, value_names, value_values
+ )
+ else:
+ return self._simple_upsert_many_txn_emulated(
+ txn, table, key_names, key_values, value_names, value_values
+ )
+
+ def _simple_upsert_many_txn_emulated(
+ self, txn, table, key_names, key_values, value_names, value_values
+ ):
+ """
+ Upsert, many times, but without native UPSERT support or batching.
+
+ Args:
+ table (str): The table to upsert into
+ key_names (list[str]): The key column names.
+ key_values (list[list]): A list of each row's key column values.
+ value_names (list[str]): The value column names. If empty, no
+ values will be used, even if value_values is provided.
+ value_values (list[list]): A list of each row's value column values.
+ Returns:
+ None
+ """
+ # No value columns, therefore make a blank list so that the following
+ # zip() works correctly.
+ if not value_names:
+ value_values = [() for x in range(len(key_values))]
+
+ for keyv, valv in zip(key_values, value_values):
+ _keys = {x: y for x, y in zip(key_names, keyv)}
+ _vals = {x: y for x, y in zip(value_names, valv)}
+
+ self._simple_upsert_txn_emulated(txn, table, _keys, _vals)
+
+ def _simple_upsert_many_txn_native_upsert(
+ self, txn, table, key_names, key_values, value_names, value_values
+ ):
+ """
+ Upsert, many times, using batching where possible.
+
+ Args:
+ table (str): The table to upsert into
+ key_names (list[str]): The key column names.
+ key_values (list[list]): A list of each row's key column values.
+ value_names (list[str]): The value column names. If empty, no
+ values will be used, even if value_values is provided.
+ value_values (list[list]): A list of each row's value column values.
+ Returns:
+ None
+ """
+ allnames = []
+ allnames.extend(key_names)
+ allnames.extend(value_names)
+
+ if not value_names:
+ # No value columns, therefore make a blank list so that the
+ # following zip() works correctly.
+ latter = "NOTHING"
+ value_values = [() for x in range(len(key_values))]
+ else:
+ latter = (
+ "UPDATE SET " + ", ".join(k + "=EXCLUDED." + k for k in value_names)
+ )
+
+ sql = "INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (%s) DO %s" % (
+ table,
+ ", ".join(k for k in allnames),
+ ", ".join("?" for _ in allnames),
+ ", ".join(key_names),
+ latter,
+ )
+
+ args = []
+
+ for x, y in zip(key_values, value_values):
+ args.append(tuple(x) + tuple(y))
+
+ return txn.execute_batch(sql, args)
+
def _simple_select_one(self, table, keyvalues, retcols,
allow_none=False, desc="_simple_select_one"):
"""Executes a SELECT query on the named table, which is expected to
diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py
index 52eb05bfbf..dd49a14524 100644
--- a/tests/storage/test__base.py
+++ b/tests/storage/test__base.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2019 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -314,3 +315,90 @@ class CacheDecoratorTestCase(unittest.TestCase):
self.assertEquals(callcount[0], 2)
self.assertEquals(callcount2[0], 3)
+
+
+class UpsertManyTests(unittest.HomeserverTestCase):
+ def prepare(self, reactor, clock, hs):
+ self.storage = hs.get_datastore()
+
+ self.table_name = "table_" + hs.get_secrets().token_hex(6)
+ self.get_success(
+ self.storage.runInteraction(
+ "create",
+ lambda x, *a: x.execute(*a),
+ "CREATE TABLE %s (id INTEGER, username TEXT, value TEXT)"
+ % (self.table_name,),
+ )
+ )
+ self.get_success(
+ self.storage.runInteraction(
+ "index",
+ lambda x, *a: x.execute(*a),
+ "CREATE UNIQUE INDEX %sindex ON %s(id, username)"
+ % (self.table_name, self.table_name),
+ )
+ )
+
+ def _dump_to_tuple(self, res):
+ for i in res:
+ yield (i["id"], i["username"], i["value"])
+
+ def test_upsert_many(self):
+ """
+ Upsert_many will perform the upsert operation across a batch of data.
+ """
+ # Add some data to an empty table
+ key_names = ["id", "username"]
+ value_names = ["value"]
+ key_values = [[1, "user1"], [2, "user2"]]
+ value_values = [["hello"], ["there"]]
+
+ self.get_success(
+ self.storage.runInteraction(
+ "test",
+ self.storage._simple_upsert_many_txn,
+ self.table_name,
+ key_names,
+ key_values,
+ value_names,
+ value_values,
+ )
+ )
+
+ # Check results are what we expect
+ res = self.get_success(
+ self.storage._simple_select_list(
+ self.table_name, None, ["id, username, value"]
+ )
+ )
+ self.assertEqual(
+ set(self._dump_to_tuple(res)),
+ set([(1, "user1", "hello"), (2, "user2", "there")]),
+ )
+
+ # Update only user2
+ key_values = [[2, "user2"]]
+ value_values = [["bleb"]]
+
+ self.get_success(
+ self.storage.runInteraction(
+ "test",
+ self.storage._simple_upsert_many_txn,
+ self.table_name,
+ key_names,
+ key_values,
+ value_names,
+ value_values,
+ )
+ )
+
+ # Check results are what we expect
+ res = self.get_success(
+ self.storage._simple_select_list(
+ self.table_name, None, ["id, username, value"]
+ )
+ )
+ self.assertEqual(
+ set(self._dump_to_tuple(res)),
+ set([(1, "user1", "hello"), (2, "user2", "bleb")]),
+ )
|
