diff options
| author | Michael Kaye <1917473+michaelkaye@users.noreply.github.com> | 2021-11-26 14:05:20 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-11-26 14:05:20 +0000 |
| commit | e2c300e7e4647500b0ff5ada8018cc984b78f91c (patch) | |
| tree | 391a9f8eab79b9ed711e48035b0f34c31e567e6a | |
| parent | Track ongoing event fetches correctly (again) (#11376) (diff) | |
| download | synapse-e2c300e7e4647500b0ff5ada8018cc984b78f91c.tar.xz | |
Create healthcheck script for synapse-workers container (#11429)
The intent is to iterate through all the worker ports and only report healthy when all are healthy, starting with the main process.
| -rw-r--r-- | changelog.d/11429.docker | 1 | ||||
| -rw-r--r-- | docker/Dockerfile-workers | 3 | ||||
| -rw-r--r-- | docker/conf-workers/healthcheck.sh.j2 | 6 | ||||
| -rwxr-xr-x | docker/configure_workers_and_start.py | 13 |
4 files changed, 23 insertions, 0 deletions
diff --git a/changelog.d/11429.docker b/changelog.d/11429.docker new file mode 100644 index 0000000000..81db719ed6 --- /dev/null +++ b/changelog.d/11429.docker @@ -0,0 +1 @@ +Update `Dockerfile-workers` to healthcheck all workers in container. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 969cf97286..46f2e17382 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -21,3 +21,6 @@ VOLUME ["/data"] # files to run the desired worker configuration. Will start supervisord. COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py ENTRYPOINT ["/configure_workers_and_start.py"] + +HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ + CMD /bin/sh /healthcheck.sh diff --git a/docker/conf-workers/healthcheck.sh.j2 b/docker/conf-workers/healthcheck.sh.j2 new file mode 100644 index 0000000000..79c621f89c --- /dev/null +++ b/docker/conf-workers/healthcheck.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/sh +# This healthcheck script is designed to return OK when every +# host involved returns OK +{%- for healthcheck_url in healthcheck_urls %} +curl -fSs {{ healthcheck_url }} || exit 1 +{%- endfor %} diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index f4ac1c22a4..adbb551cee 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -474,10 +474,16 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Determine the load-balancing upstreams to configure nginx_upstream_config = "" + + # At the same time, prepare a list of internal endpoints to healthcheck + # starting with the main process which exists even if no workers do. + healthcheck_urls = ["http://localhost:8080/health"] + for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items(): body = "" for port in upstream_worker_ports: body += " server localhost:%d;\n" % (port,) + healthcheck_urls.append("http://localhost:%d/health" % (port,)) # Add to the list of configured upstreams nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format( @@ -510,6 +516,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str): worker_config=supervisord_config, ) + # healthcheck config + convert( + "/conf/healthcheck.sh.j2", + "/healthcheck.sh", + healthcheck_urls=healthcheck_urls, + ) + # Ensure the logging directory exists log_dir = data_dir + "/logs" if not os.path.exists(log_dir): |