From aa6ad288f16ed263b2a94b480259639d52ff6cad Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 24 Jan 2020 11:01:57 +0200 Subject: Clarifications to the workers documentation * Add note that user_dir requires disabling user dir updates from the main synapse process. * Add note that federation_reader should have the federation listener resource. Signed-off-by: Jason Robinson --- docs/workers.md | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'docs') diff --git a/docs/workers.md b/docs/workers.md index 0ab269fd96..a5d6d18f23 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -185,6 +185,9 @@ reverse-proxy configuration. The `^/_matrix/federation/v1/send/` endpoint must only be handled by a single instance. +Note that the `worker_listeners.resources.name` needs to be set to `federation` +for this worker. + ### `synapse.app.federation_sender` Handles sending federation traffic to other servers. Doesn't handle any @@ -265,6 +268,10 @@ the following regular expressions: ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ +When using this worker you must also set `update_user_directory: False` in the +shared configuration file to stop the main synapse running background +jobs related to updating the user directory. + ### `synapse.app.frontend_proxy` Proxies some frequently-requested client endpoints to add caching and remove -- cgit 1.5.1 From 1fe50013691873e7549276fcc7a679bfe19595f9 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 27 Jan 2020 10:20:48 +0200 Subject: Fix federation_reader listeners doc as per PR review Signed-off-by: Jason Robinson --- docs/workers.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/workers.md b/docs/workers.md index a5d6d18f23..09a9d8a7b8 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -185,8 +185,18 @@ reverse-proxy configuration. The `^/_matrix/federation/v1/send/` endpoint must only be handled by a single instance. -Note that the `worker_listeners.resources.name` needs to be set to `federation` -for this worker. +Note that `federation` must be added to the listener resources in the worker config: + +```yaml +worker_app: synapse.app.federation_reader +... +worker_listeners: + - type: http + port: + resources: + - names: + - federation +``` ### `synapse.app.federation_sender` -- cgit 1.5.1 From 02b44db922f01a35787d2535a834c9774b68020b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Jan 2020 13:44:21 +0000 Subject: Warn if postgres database has non-C locale. (#6734) As using non-C locale can cause issues on upgrading OS. --- UPGRADE.rst | 9 ++++++++ changelog.d/6734.bugfix | 1 + docs/postgres.md | 20 +++++++++++++++++- synapse/storage/engines/postgres.py | 42 +++++++++++++++++++++++++++++++++++++ synapse/storage/engines/sqlite.py | 5 +++++ synapse/storage/prepare_database.py | 5 +++++ 6 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 changelog.d/6734.bugfix (limited to 'docs') diff --git a/UPGRADE.rst b/UPGRADE.rst index a0202932b1..470246f128 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -76,6 +76,15 @@ for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb +Upgrading to **** +=============================== + +Synapse will now log a warning on start up if used with a PostgreSQL database +that has a non-recommended locale set. + +See [docs/postgres.md](docs/postgres.md) for details. + + Upgrading to v1.8.0 =================== diff --git a/changelog.d/6734.bugfix b/changelog.d/6734.bugfix new file mode 100644 index 0000000000..79c6bab4d1 --- /dev/null +++ b/changelog.d/6734.bugfix @@ -0,0 +1 @@ +Warn if postgres database has a non-C locale, as that can cause issues when upgrading locales (e.g. due to upgrading OS). diff --git a/docs/postgres.md b/docs/postgres.md index 7cb1ad18d4..e0793ecee8 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -32,7 +32,7 @@ Assuming your PostgreSQL database user is called `postgres`, first authenticate su - postgres # Or, if your system uses sudo to get administrative rights sudo -u postgres bash - + Then, create a user ``synapse_user`` with: createuser --pwprompt synapse_user @@ -63,6 +63,24 @@ You may need to enable password authentication so `synapse_user` can connect to the database. See . +### Fixing incorrect `COLLATE` or `CTYPE` + +Synapse will refuse to set up a new database if it has the wrong values of +`COLLATE` and `CTYPE` set, and will log warnings on existing databases. Using +different locales can cause issues if the locale library is updated from +underneath the database, or if a different version of the locale is used on any +replicas. + +The safest way to fix the issue is to take a dump and recreate the database with +the correct `COLLATE` and `CTYPE` parameters (as per +[docs/postgres.md](docs/postgres.md)). It is also possible to change the +parameters on a live database and run a `REINDEX` on the entire database, +however extreme care must be taken to avoid database corruption. + +Note that the above may fail with an error about duplicate rows if corruption +has already occurred, and such duplicate rows will need to be manually removed. + + ## Tuning Postgres The default settings should be fine for most deployments. For larger diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index c84cb452b0..a077345960 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging + from ._base import IncorrectDatabaseSetup +logger = logging.getLogger(__name__) + class PostgresEngine(object): single_threaded = False @@ -52,6 +56,44 @@ class PostgresEngine(object): "See docs/postgres.rst for more information." % (rows[0][0],) ) + txn.execute( + "SELECT datcollate, datctype FROM pg_database WHERE datname = current_database()" + ) + collation, ctype = txn.fetchone() + if collation != "C": + logger.warning( + "Database has incorrect collation of %r. Should be 'C'", collation + ) + + if ctype != "C": + logger.warning( + "Database has incorrect ctype of %r. Should be 'C'", ctype + ) + + def check_new_database(self, txn): + """Gets called when setting up a brand new database. This allows us to + apply stricter checks on new databases versus existing database. + """ + + txn.execute( + "SELECT datcollate, datctype FROM pg_database WHERE datname = current_database()" + ) + collation, ctype = txn.fetchone() + + errors = [] + + if collation != "C": + errors.append(" - 'COLLATE' is set to %r. Should be 'C'" % (collation,)) + + if ctype != "C": + errors.append(" - 'CTYPE' is set to %r. Should be 'C'" % (collation,)) + + if errors: + raise IncorrectDatabaseSetup( + "Database is incorrectly configured:\n\n%s\n\n" + "See docs/postgres.md for more information." % ("\n".join(errors)) + ) + def convert_param_style(self, sql): return sql.replace("?", "%s") diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index cbf52f5191..641e490697 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -59,6 +59,11 @@ class Sqlite3Engine(object): if version < (3, 11, 0): raise RuntimeError("Synapse requires sqlite 3.11 or above.") + def check_new_database(self, txn): + """Gets called when setting up a brand new database. This allows us to + apply stricter checks on new databases versus existing database. + """ + def convert_param_style(self, sql): return sql diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index e86984cd50..c285ef52a0 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -136,6 +136,11 @@ def _setup_new_database(cur, database_engine, data_stores): data_stores (list[str]): The names of the data stores to instantiate on the given database. """ + + # We're about to set up a brand new database so we check that its + # configured to our liking. + database_engine.check_new_database(cur) + current_dir = os.path.join(dir_path, "schema", "full_schemas") directory_entries = os.listdir(current_dir) -- cgit 1.5.1