diff --git a/changelog.d/11178.feature b/changelog.d/11178.feature
new file mode 100644
index 0000000000..10b1cdffdc
--- /dev/null
+++ b/changelog.d/11178.feature
@@ -0,0 +1 @@
+Add metrics for thread pool usage.
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index f4c3f867a8..f2c1028b5d 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -45,6 +45,7 @@ from synapse.events.spamcheck import load_legacy_spam_checkers
from synapse.events.third_party_rules import load_legacy_third_party_event_rules
from synapse.handlers.auth import load_legacy_password_auth_providers
from synapse.logging.context import PreserveLoggingContext
+from synapse.metrics import register_threadpool
from synapse.metrics.background_process_metrics import wrap_as_background_process
from synapse.metrics.jemalloc import setup_jemalloc_stats
from synapse.util.caches.lrucache import setup_expire_lru_cache_entries
@@ -351,6 +352,10 @@ async def start(hs: "HomeServer"):
GAIResolver(reactor, getThreadPool=lambda: resolver_threadpool)
)
+ # Register the threadpools with our metrics.
+ register_threadpool("default", reactor.getThreadPool())
+ register_threadpool("gai_resolver", resolver_threadpool)
+
# Set up the SIGHUP machinery.
if hasattr(signal, "SIGHUP"):
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index e902109af3..91ee5c8193 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -32,6 +32,7 @@ from prometheus_client.core import (
)
from twisted.internet import reactor
+from twisted.python.threadpool import ThreadPool
import synapse
from synapse.metrics._exposition import (
@@ -526,6 +527,42 @@ threepid_send_requests = Histogram(
labelnames=("type", "reason"),
)
+threadpool_total_threads = Gauge(
+ "synapse_threadpool_total_threads",
+ "Total number of threads currently in the threadpool",
+ ["name"],
+)
+
+threadpool_total_working_threads = Gauge(
+ "synapse_threadpool_working_threads",
+ "Number of threads currently working in the threadpool",
+ ["name"],
+)
+
+threadpool_total_min_threads = Gauge(
+ "synapse_threadpool_min_threads",
+ "Minimum number of threads configured in the threadpool",
+ ["name"],
+)
+
+threadpool_total_max_threads = Gauge(
+ "synapse_threadpool_max_threads",
+ "Maximum number of threads configured in the threadpool",
+ ["name"],
+)
+
+
+def register_threadpool(name: str, threadpool: ThreadPool) -> None:
+ """Add metrics for the threadpool."""
+
+ threadpool_total_min_threads.labels(name).set(threadpool.min)
+ threadpool_total_max_threads.labels(name).set(threadpool.max)
+
+ threadpool_total_threads.labels(name).set_function(lambda: len(threadpool.threads))
+ threadpool_total_working_threads.labels(name).set_function(
+ lambda: len(threadpool.working)
+ )
+
class ReactorLastSeenMetric:
def collect(self):
diff --git a/synapse/storage/database.py b/synapse/storage/database.py
index fa4e89d35c..5c71e27518 100644
--- a/synapse/storage/database.py
+++ b/synapse/storage/database.py
@@ -48,6 +48,7 @@ from synapse.logging.context import (
current_context,
make_deferred_yieldable,
)
+from synapse.metrics import register_threadpool
from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.storage.background_updates import BackgroundUpdater
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
@@ -104,13 +105,17 @@ def make_pool(
LoggingDatabaseConnection(conn, engine, "on_new_connection")
)
- return adbapi.ConnectionPool(
+ connection_pool = adbapi.ConnectionPool(
db_config.config["name"],
cp_reactor=reactor,
cp_openfun=_on_new_connection,
**db_args,
)
+ register_threadpool(f"database-{db_config.name}", connection_pool.threadpool)
+
+ return connection_pool
+
def make_conn(
db_config: DatabaseConnectionConfig,
|
