From 24ab712993d40d083d970452c0e61b1ea72abbcf Mon Sep 17 00:00:00 2001
From: Stephan Behnke <stephanos@users.noreply.github.com>
Date: Tue, 17 Feb 2026 11:27:55 -0800
Subject: [PATCH] Docker compose healthchecks (#9335)

## What changed?

Add health checks for docker compose dependencies.

## Why?

Seeing connection timeouts for cass_es8
[[example](https://github.com/temporalio/temporal/actions/runs/22005015596/job/63897748984?pr=9292)]
that are failing tests.

## How did you test it?
- [ ] built
- [ ] run locally and tested manually
- [x] covered by existing tests
- [ ] added new unit test(s)
- [ ] added new functional test(s)

## Potential risks

Might make tests a little bit slower due to extra wait time.
---
 .github/workflows/run-tests.yml   | 13 +++++++++++++
 develop/github/docker-compose.yml | 26 ++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 8d63dac6ef..de329e088b 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -401,6 +401,12 @@ jobs:
           path: ~/.cache/go-build
           key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
 
+      - name: Wait for containerized dependencies to be healthy
+        run: |
+          # Word splitting is intentional here.
+          # shellcheck disable=SC2046
+          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up --wait $(docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} ps --services)
+
       - name: Run integration test
         timeout-minutes: 15
         run: ./develop/github/monitor_test.sh make integration-test-coverage
@@ -516,6 +522,13 @@ jobs:
       - name: ${{ matrix.display_name == 'smoke' && 'ℹ️ Smoke test' || 'ℹ️ Full test' }}
         run: echo "::notice::${{ matrix.display_name == 'smoke' && 'This is a smoke test. Add the test-all-dbs label to run all tests on all DBs.' || needs.test-setup.outputs.full_test_reason }}"
 
+      - name: Wait for containerized dependencies to be healthy
+        if: ${{ toJson(matrix.containers) != '[]' }}
+        run: |
+          # Word splitting is intentional here.
+          # shellcheck disable=SC2046
+          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up --wait $(docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} ps --services)
+
       - name: Run functional test
         timeout-minutes: ${{ matrix.github_timeout }}
         run: ./develop/github/monitor_test.sh ${{ matrix.cmd }}
diff --git a/develop/github/docker-compose.yml b/develop/github/docker-compose.yml
index ee5b73ea5d..6cf535596e 100644
--- a/develop/github/docker-compose.yml
+++ b/develop/github/docker-compose.yml
@@ -1,3 +1,8 @@
+x-healthcheck-defaults: &healthcheck-defaults
+  interval: 3s
+  timeout: 3s
+  retries: 30
+
 services:
   cassandra:
     image: cassandra:3.11
@@ -9,6 +14,9 @@ services:
       HEAP_NEWSIZE: "200M"
       # Increase native transport threads for handling more concurrent connections
       JVM_EXTRA_OPTS: "-Dcassandra.native_transport_max_threads=512"
+    healthcheck:
+      !!merge <<: *healthcheck-defaults
+      test: ["CMD-SHELL", "cqlsh -e 'describe cluster'"]
 
   mysql:
     image: mysql:8.0.29-oracle
@@ -19,6 +27,9 @@ services:
     command: --max-connections=500
     volumes:
       - ./mysql-init:/docker-entrypoint-initdb.d
+    healthcheck:
+      !!merge <<: *healthcheck-defaults
+      test: ["CMD-SHELL", "mysqladmin ping -h localhost -uroot -proot"]
 
   postgresql:
     image: postgres:13.5
@@ -30,6 +41,9 @@ services:
     command: postgres -c max_connections=500
     volumes:
       - ./postgresql-init:/docker-entrypoint-initdb.d
+    healthcheck:
+      !!merge <<: *healthcheck-defaults
+      test: ["CMD-SHELL", "pg_isready -U temporal"]
 
   elasticsearch:
     image: elasticsearch:7.10.1
@@ -42,6 +56,9 @@ services:
       - cluster.routing.allocation.disk.watermark.flood_stage=128mb
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1g -Xmx1g
+    healthcheck:
+      !!merge <<: *healthcheck-defaults
+      test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
 
   elasticsearch8:
     image: elasticsearch:8.5.0
@@ -55,6 +72,9 @@ services:
       - discovery.type=single-node
       - xpack.security.enabled=false
       - ES_JAVA_OPTS=-Xms1g -Xmx1g
+    healthcheck:
+      !!merge <<: *healthcheck-defaults
+      test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
 
   opensearch2:
     image: opensearchproject/opensearch:2
@@ -68,6 +88,9 @@ services:
       - discovery.type=single-node
       - DISABLE_SECURITY_PLUGIN=true
       - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
+    healthcheck:
+      !!merge <<: *healthcheck-defaults
+      test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
 
   opensearch3:
     image: opensearchproject/opensearch:3
@@ -81,3 +104,6 @@ services:
       - discovery.type=single-node
       - DISABLE_SECURITY_PLUGIN=true
       - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
+    healthcheck:
+      !!merge <<: *healthcheck-defaults
+      test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]