diff --git a/synapse/_scripts/generate_workers_map.py b/synapse/_scripts/generate_workers_map.py
index 715c7ddc17..09feb8cf30 100755
--- a/synapse/_scripts/generate_workers_map.py
+++ b/synapse/_scripts/generate_workers_map.py
@@ -171,7 +171,7 @@ def elide_http_methods_if_unconflicting(
"""
def paths_to_methods_dict(
- methods_and_paths: Iterable[Tuple[str, str]]
+ methods_and_paths: Iterable[Tuple[str, str]],
) -> Dict[str, Set[str]]:
"""
Given (method, path) pairs, produces a dict from path to set of methods
@@ -201,7 +201,7 @@ def elide_http_methods_if_unconflicting(
def simplify_path_regexes(
- registrations: Dict[Tuple[str, str], EndpointDescription]
+ registrations: Dict[Tuple[str, str], EndpointDescription],
) -> Dict[Tuple[str, str], EndpointDescription]:
"""
Simplify all the path regexes for the dict of endpoint descriptions,
diff --git a/synapse/_scripts/hash_password.py b/synapse/_scripts/hash_password.py
index 3bed367be2..2b7d3585cb 100755
--- a/synapse/_scripts/hash_password.py
+++ b/synapse/_scripts/hash_password.py
@@ -56,7 +56,9 @@ def main() -> None:
password_pepper = password_config.get("pepper", password_pepper)
password = args.password
- if not password:
+ if not password and not sys.stdin.isatty():
+ password = sys.stdin.readline().strip()
+ elif not password:
password = prompt_for_pass()
# On Python 2, make sure we decode it to Unicode before we normalise it
diff --git a/synapse/_scripts/review_recent_signups.py b/synapse/_scripts/review_recent_signups.py
index ad88df477a..03bd58a1a1 100644
--- a/synapse/_scripts/review_recent_signups.py
+++ b/synapse/_scripts/review_recent_signups.py
@@ -40,6 +40,7 @@ from synapse.storage.engines import create_engine
class ReviewConfig(RootConfig):
"A config class that just pulls out the database config"
+
config_classes = [DatabaseConfig]
@@ -73,13 +74,6 @@ def get_recent_users(
user_infos = [UserInfo(user_id, creation_ts) for user_id, creation_ts in txn]
for user_info in user_infos:
- user_info.emails = DatabasePool.simple_select_onecol_txn(
- txn,
- table="user_threepids",
- keyvalues={"user_id": user_info.user_id, "medium": "email"},
- retcol="address",
- )
-
sql = """
SELECT room_id, canonical_alias, name, join_rules
FROM local_current_membership
@@ -160,7 +154,11 @@ def main() -> None:
with make_conn(database_config, engine, "review_recent_signups") as db_conn:
# This generates a type of Cursor, not LoggingTransaction.
- user_infos = get_recent_users(db_conn.cursor(), since_ms, exclude_users_with_appservice) # type: ignore[arg-type]
+ user_infos = get_recent_users(
+ db_conn.cursor(),
+ since_ms, # type: ignore[arg-type]
+ exclude_users_with_appservice,
+ )
for user_info in user_infos:
if exclude_users_with_email and user_info.emails:
diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py
index 5c6db8118f..573c70696e 100755
--- a/synapse/_scripts/synapse_port_db.py
+++ b/synapse/_scripts/synapse_port_db.py
@@ -42,12 +42,12 @@ from typing import (
Set,
Tuple,
Type,
+ TypedDict,
TypeVar,
cast,
)
import yaml
-from typing_extensions import TypedDict
from twisted.internet import defer, reactor as reactor_
@@ -88,6 +88,7 @@ from synapse.storage.databases.main.relations import RelationsWorkerStore
from synapse.storage.databases.main.room import RoomBackgroundUpdateStore
from synapse.storage.databases.main.roommember import RoomMemberBackgroundUpdateStore
from synapse.storage.databases.main.search import SearchBackgroundUpdateStore
+from synapse.storage.databases.main.sliding_sync import SlidingSyncStore
from synapse.storage.databases.main.state import MainStateBackgroundUpdateStore
from synapse.storage.databases.main.stats import StatsStore
from synapse.storage.databases.main.user_directory import (
@@ -127,8 +128,14 @@ BOOLEAN_COLUMNS = {
"pushers": ["enabled"],
"redactions": ["have_censored"],
"remote_media_cache": ["authenticated"],
+ "room_memberships": ["participant"],
"room_stats_state": ["is_federatable"],
"rooms": ["is_public", "has_auth_chain_index"],
+ "sliding_sync_joined_rooms": ["is_encrypted"],
+ "sliding_sync_membership_snapshots": [
+ "has_known_state",
+ "is_encrypted",
+ ],
"users": ["shadow_banned", "approved", "locked", "suspended"],
"un_partial_stated_event_stream": ["rejection_status_changed"],
"users_who_share_rooms": ["share_private"],
@@ -185,6 +192,11 @@ APPEND_ONLY_TABLES = [
IGNORED_TABLES = {
+ # Porting the auto generated sequence in this table is non-trivial.
+ # None of the entries in this list are mandatory for Synapse to keep working.
+ # If state group disk space is an issue after the port, the
+ # `mark_unreferenced_state_groups_for_deletion_bg_update` background task can be run again.
+ "state_groups_pending_deletion",
# We don't port these tables, as they're a faff and we can regenerate
# them anyway.
"user_directory",
@@ -210,6 +222,15 @@ IGNORED_TABLES = {
}
+# These background updates will not be applied upon creation of the postgres database.
+IGNORED_BACKGROUND_UPDATES = {
+ # Reapplying this background update to the postgres database is unnecessary after
+ # already having waited for the SQLite database to complete all running background
+ # updates.
+ "mark_unreferenced_state_groups_for_deletion_bg_update",
+}
+
+
# Error returned by the run function. Used at the top-level part of the script to
# handle errors and return codes.
end_error: Optional[str] = None
@@ -250,6 +271,7 @@ class Store(
ReceiptsBackgroundUpdateStore,
RelationsWorkerStore,
EventFederationWorkerStore,
+ SlidingSyncStore,
):
def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
@@ -680,6 +702,20 @@ class Porter:
# 0 means off. 1 means full. 2 means incremental.
return autovacuum_setting != 0
+ async def remove_ignored_background_updates_from_database(self) -> None:
+ def _remove_delete_unreferenced_state_groups_bg_updates(
+ txn: LoggingTransaction,
+ ) -> None:
+ txn.execute(
+ "DELETE FROM background_updates WHERE update_name = ANY(?)",
+ (list(IGNORED_BACKGROUND_UPDATES),),
+ )
+
+ await self.postgres_store.db_pool.runInteraction(
+ "remove_delete_unreferenced_state_groups_bg_updates",
+ _remove_delete_unreferenced_state_groups_bg_updates,
+ )
+
async def run(self) -> None:
"""Ports the SQLite database to a PostgreSQL database.
@@ -712,9 +748,7 @@ class Porter:
return
# Check if all background updates are done, abort if not.
- updates_complete = (
- await self.sqlite_store.db_pool.updates.has_completed_background_updates()
- )
+ updates_complete = await self.sqlite_store.db_pool.updates.has_completed_background_updates()
if not updates_complete:
end_error = (
"Pending background updates exist in the SQLite3 database."
@@ -727,6 +761,8 @@ class Porter:
self.hs_config.database.get_single_database()
)
+ await self.remove_ignored_background_updates_from_database()
+
await self.run_background_updates_on_postgres()
self.progress.set_state("Creating port tables")
@@ -1029,7 +1065,7 @@ class Porter:
def get_sent_table_size(txn: LoggingTransaction) -> int:
txn.execute(
- "SELECT count(*) FROM sent_transactions" " WHERE ts >= ?", (yesterday,)
+ "SELECT count(*) FROM sent_transactions WHERE ts >= ?", (yesterday,)
)
result = txn.fetchone()
assert result is not None
@@ -1090,10 +1126,10 @@ class Porter:
return done, remaining + done
async def _setup_state_group_id_seq(self) -> None:
- curr_id: Optional[int] = (
- await self.sqlite_store.db_pool.simple_select_one_onecol(
- table="state_groups", keyvalues={}, retcol="MAX(id)", allow_none=True
- )
+ curr_id: Optional[
+ int
+ ] = await self.sqlite_store.db_pool.simple_select_one_onecol(
+ table="state_groups", keyvalues={}, retcol="MAX(id)", allow_none=True
)
if not curr_id:
@@ -1181,13 +1217,13 @@ class Porter:
)
async def _setup_auth_chain_sequence(self) -> None:
- curr_chain_id: Optional[int] = (
- await self.sqlite_store.db_pool.simple_select_one_onecol(
- table="event_auth_chains",
- keyvalues={},
- retcol="MAX(chain_id)",
- allow_none=True,
- )
+ curr_chain_id: Optional[
+ int
+ ] = await self.sqlite_store.db_pool.simple_select_one_onecol(
+ table="event_auth_chains",
+ keyvalues={},
+ retcol="MAX(chain_id)",
+ allow_none=True,
)
def r(txn: LoggingTransaction) -> None:
diff --git a/synapse/_scripts/synctl.py b/synapse/_scripts/synctl.py
index 688df9485c..2e2aa27a17 100755
--- a/synapse/_scripts/synctl.py
+++ b/synapse/_scripts/synctl.py
@@ -292,9 +292,9 @@ def main() -> None:
for key in worker_config:
if key == "worker_app": # But we allow worker_app
continue
- assert not key.startswith(
- "worker_"
- ), "Main process cannot use worker_* config"
+ assert not key.startswith("worker_"), (
+ "Main process cannot use worker_* config"
+ )
else:
worker_pidfile = worker_config["worker_pid_file"]
worker_cache_factor = worker_config.get("synctl_cache_factor")
|
