Require mypy for synapse/ & tests/ unless excluded (#11282)

Signed-off-by: Dan Callahan <danc@element.io>

3 files changed, 168 insertions, 77 deletions

diff --git a/changelog.d/11282.misc b/changelog.d/11282.misc
new file mode 100644
index 0000000000..4720519cbc
--- /dev/null
+++ b/changelog.d/11282.misc
@@ -0,0 +1 @@
+Require all files in synapse/ and tests/ to pass mypy unless specifically excluded.
diff --git a/mypy.ini b/mypy.ini
index 600402a5d3..8c20c293aa 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -10,82 +10,173 @@ warn_unreachable = True
 local_partial_types = True
 no_implicit_optional = True
 
-# To find all folders that pass mypy you run:
-#
-#   find synapse/* -type d -not -name __pycache__ -exec bash -c "mypy '{}' > /dev/null"  \; -print
-
 files =
   scripts-dev/sign_json,
-  synapse/__init__.py,
-  synapse/api,
-  synapse/appservice,
-  synapse/config,
-  synapse/crypto,
-  synapse/event_auth.py,
-  synapse/events,
-  synapse/federation,
-  synapse/groups,
-  synapse/handlers,
-  synapse/http,
-  synapse/logging,
-  synapse/metrics,
-  synapse/module_api,
-  synapse/notifier.py,
-  synapse/push,
-  synapse/replication,
-  synapse/rest,
-  synapse/server.py,
-  synapse/server_notices,
-  synapse/spam_checker_api,
-  synapse/state,
-  synapse/storage/__init__.py,
-  synapse/storage/_base.py,
-  synapse/storage/background_updates.py,
-  synapse/storage/databases/main/appservice.py,
-  synapse/storage/databases/main/client_ips.py,
-  synapse/storage/databases/main/events.py,
-  synapse/storage/databases/main/keys.py,
-  synapse/storage/databases/main/pusher.py,
-  synapse/storage/databases/main/registration.py,
-  synapse/storage/databases/main/relations.py,
-  synapse/storage/databases/main/session.py,
-  synapse/storage/databases/main/stream.py,
-  synapse/storage/databases/main/ui_auth.py,
-  synapse/storage/databases/state,
-  synapse/storage/database.py,
-  synapse/storage/engines,
-  synapse/storage/keys.py,
-  synapse/storage/persist_events.py,
-  synapse/storage/prepare_database.py,
-  synapse/storage/purge_events.py,
-  synapse/storage/push_rule.py,
-  synapse/storage/relations.py,
-  synapse/storage/roommember.py,
-  synapse/storage/state.py,
-  synapse/storage/types.py,
-  synapse/storage/util,
-  synapse/streams,
-  synapse/types.py,
-  synapse/util,
-  synapse/visibility.py,
-  tests/replication,
-  tests/test_event_auth.py,
-  tests/test_utils,
-  tests/handlers/test_password_providers.py,
-  tests/handlers/test_room.py,
-  tests/handlers/test_room_summary.py,
-  tests/handlers/test_send_email.py,
-  tests/handlers/test_sync.py,
-  tests/handlers/test_user_directory.py,
-  tests/rest/client/test_login.py,
-  tests/rest/client/test_auth.py,
-  tests/rest/client/test_relations.py,
-  tests/rest/media/v1/test_filepath.py,
-  tests/rest/media/v1/test_oembed.py,
-  tests/storage/test_state.py,
-  tests/storage/test_user_directory.py,
-  tests/util/test_itertools.py,
-  tests/util/test_stream_change_cache.py
+  setup.py,
+  synapse/,
+  tests/
+
+# Note: Better exclusion syntax coming in mypy > 0.910
+# https://github.com/python/mypy/pull/11329
+#
+# For now, set the (?x) flag enable "verbose" regexes
+# https://docs.python.org/3/library/re.html#re.X
+exclude = (?x)
+  ^(
+   |synapse/_scripts/register_new_matrix_user.py
+   |synapse/_scripts/review_recent_signups.py
+   |synapse/app/__init__.py
+   |synapse/app/_base.py
+   |synapse/app/admin_cmd.py
+   |synapse/app/appservice.py
+   |synapse/app/client_reader.py
+   |synapse/app/event_creator.py
+   |synapse/app/federation_reader.py
+   |synapse/app/federation_sender.py
+   |synapse/app/frontend_proxy.py
+   |synapse/app/generic_worker.py
+   |synapse/app/homeserver.py
+   |synapse/app/media_repository.py
+   |synapse/app/phone_stats_home.py
+   |synapse/app/pusher.py
+   |synapse/app/synchrotron.py
+   |synapse/app/user_dir.py
+   |synapse/storage/databases/__init__.py
+   |synapse/storage/databases/main/__init__.py
+   |synapse/storage/databases/main/account_data.py
+   |synapse/storage/databases/main/cache.py
+   |synapse/storage/databases/main/censor_events.py
+   |synapse/storage/databases/main/deviceinbox.py
+   |synapse/storage/databases/main/devices.py
+   |synapse/storage/databases/main/directory.py
+   |synapse/storage/databases/main/e2e_room_keys.py
+   |synapse/storage/databases/main/end_to_end_keys.py
+   |synapse/storage/databases/main/event_federation.py
+   |synapse/storage/databases/main/event_push_actions.py
+   |synapse/storage/databases/main/events_bg_updates.py
+   |synapse/storage/databases/main/events_forward_extremities.py
+   |synapse/storage/databases/main/events_worker.py
+   |synapse/storage/databases/main/filtering.py
+   |synapse/storage/databases/main/group_server.py
+   |synapse/storage/databases/main/lock.py
+   |synapse/storage/databases/main/media_repository.py
+   |synapse/storage/databases/main/metrics.py
+   |synapse/storage/databases/main/monthly_active_users.py
+   |synapse/storage/databases/main/openid.py
+   |synapse/storage/databases/main/presence.py
+   |synapse/storage/databases/main/profile.py
+   |synapse/storage/databases/main/purge_events.py
+   |synapse/storage/databases/main/push_rule.py
+   |synapse/storage/databases/main/receipts.py
+   |synapse/storage/databases/main/rejections.py
+   |synapse/storage/databases/main/room.py
+   |synapse/storage/databases/main/room_batch.py
+   |synapse/storage/databases/main/roommember.py
+   |synapse/storage/databases/main/search.py
+   |synapse/storage/databases/main/signatures.py
+   |synapse/storage/databases/main/state.py
+   |synapse/storage/databases/main/state_deltas.py
+   |synapse/storage/databases/main/stats.py
+   |synapse/storage/databases/main/tags.py
+   |synapse/storage/databases/main/transactions.py
+   |synapse/storage/databases/main/user_directory.py
+   |synapse/storage/databases/main/user_erasure_store.py
+   |synapse/storage/schema/
+
+   |tests/api/test_auth.py
+   |tests/api/test_ratelimiting.py
+   |tests/app/test_openid_listener.py
+   |tests/appservice/test_scheduler.py
+   |tests/config/test_cache.py
+   |tests/config/test_tls.py
+   |tests/crypto/test_keyring.py
+   |tests/events/test_presence_router.py
+   |tests/events/test_utils.py
+   |tests/federation/test_federation_catch_up.py
+   |tests/federation/test_federation_sender.py
+   |tests/federation/test_federation_server.py
+   |tests/federation/transport/test_knocking.py
+   |tests/federation/transport/test_server.py
+   |tests/handlers/test_cas.py
+   |tests/handlers/test_directory.py
+   |tests/handlers/test_e2e_keys.py
+   |tests/handlers/test_federation.py
+   |tests/handlers/test_oidc.py
+   |tests/handlers/test_presence.py
+   |tests/handlers/test_profile.py
+   |tests/handlers/test_saml.py
+   |tests/handlers/test_typing.py
+   |tests/http/federation/test_matrix_federation_agent.py
+   |tests/http/federation/test_srv_resolver.py
+   |tests/http/test_fedclient.py
+   |tests/http/test_proxyagent.py
+   |tests/http/test_servlet.py
+   |tests/http/test_site.py
+   |tests/logging/__init__.py
+   |tests/logging/test_terse_json.py
+   |tests/module_api/test_api.py
+   |tests/push/test_email.py
+   |tests/push/test_http.py
+   |tests/push/test_presentable_names.py
+   |tests/push/test_push_rule_evaluator.py
+   |tests/rest/admin/test_admin.py
+   |tests/rest/admin/test_device.py
+   |tests/rest/admin/test_media.py
+   |tests/rest/admin/test_server_notice.py
+   |tests/rest/admin/test_user.py
+   |tests/rest/admin/test_username_available.py
+   |tests/rest/client/test_account.py
+   |tests/rest/client/test_events.py
+   |tests/rest/client/test_filter.py
+   |tests/rest/client/test_groups.py
+   |tests/rest/client/test_register.py
+   |tests/rest/client/test_report_event.py
+   |tests/rest/client/test_rooms.py
+   |tests/rest/client/test_third_party_rules.py
+   |tests/rest/client/test_transactions.py
+   |tests/rest/client/test_typing.py
+   |tests/rest/client/utils.py
+   |tests/rest/key/v2/test_remote_key_resource.py
+   |tests/rest/media/v1/test_base.py
+   |tests/rest/media/v1/test_media_storage.py
+   |tests/rest/media/v1/test_url_preview.py
+   |tests/scripts/test_new_matrix_user.py
+   |tests/server.py
+   |tests/server_notices/test_resource_limits_server_notices.py
+   |tests/state/test_v2.py
+   |tests/storage/test_account_data.py
+   |tests/storage/test_appservice.py
+   |tests/storage/test_background_update.py
+   |tests/storage/test_base.py
+   |tests/storage/test_client_ips.py
+   |tests/storage/test_database.py
+   |tests/storage/test_event_federation.py
+   |tests/storage/test_id_generators.py
+   |tests/storage/test_roommember.py
+   |tests/test_metrics.py
+   |tests/test_phone_home.py
+   |tests/test_server.py
+   |tests/test_state.py
+   |tests/test_terms_auth.py
+   |tests/test_visibility.py
+   |tests/unittest.py
+   |tests/util/caches/test_cached_call.py
+   |tests/util/caches/test_deferred_cache.py
+   |tests/util/caches/test_descriptors.py
+   |tests/util/caches/test_response_cache.py
+   |tests/util/caches/test_ttlcache.py
+   |tests/util/test_async_helpers.py
+   |tests/util/test_batching_queue.py
+   |tests/util/test_dict_cache.py
+   |tests/util/test_expiring_cache.py
+   |tests/util/test_file_consumer.py
+   |tests/util/test_linearizer.py
+   |tests/util/test_logcontext.py
+   |tests/util/test_lrucache.py
+   |tests/util/test_rwlock.py
+   |tests/util/test_wheel_timer.py
+   |tests/utils.py
+   )$
 
 [mypy-synapse.api.*]
 disallow_untyped_defs = True
diff --git a/setup.py b/setup.py
index 345cff09c3..5d602db240 100755
--- a/setup.py
+++ b/setup.py
@@ -17,6 +17,7 @@
 # limitations under the License.
 import glob
 import os
+from typing import Any, Dict
 
 from setuptools import Command, find_packages, setup
 
@@ -49,8 +50,6 @@ here = os.path.abspath(os.path.dirname(__file__))
 # [1]: http://tox.readthedocs.io/en/2.5.0/example/basic.html#integration-with-setup-py-test-command
 # [2]: https://pypi.python.org/pypi/setuptools_trial
 class TestCommand(Command):
-    user_options = []
-
     def initialize_options(self):
         pass
 
@@ -75,7 +74,7 @@ def read_file(path_segments):
 
 def exec_file(path_segments):
     """Execute a single python file to get the variables defined in it"""
-    result = {}
+    result: Dict[str, Any] = {}
     code = read_file(path_segments)
     exec(code, result)
     return result