temporal/.github/workflows/run-tests.yml

name: All Tests

on:
  pull_request:
  push:
    branches:
      - main
      - cloud/*
      - feature/*
      - release/*

permissions:
  contents: read

concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
  group: run-tests-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  # For pull_request: use the head of the PR branch (not the merge branch which is the default!)
  # For push: use the pushed commit.
  COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
  PR_BASE_COMMIT: ${{ github.event.pull_request.base.sha }}
  DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
  TEMPORAL_VERSION_CHECK_DISABLED: 1
  TEMPORAL_AWAIT_ATTEMPT_TIMEOUT: 15s
  MAX_TEST_ATTEMPTS: 3
  SHARD_COUNT: 5 # NOTE: must match shard count in optimize-test-sharding.yml

jobs:
  test-setup:
    name: Test setup
    runs-on: ubuntu-latest
    outputs:
      job_matrix: ${{ steps.build_matrix.outputs.job_matrix }}
      full_test_reason: ${{ steps.determine_scope.outputs.full_test_reason }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ env.COMMIT }}
          fetch-depth: 0

      - name: Fetch base branch
        if: ${{ github.event_name == 'pull_request' }}
        run: git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}

      - name: Compute merge base
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          MERGE_BASE="$(git merge-base "${{ env.COMMIT }}" "${{ github.event.pull_request.base.ref }}")"
          echo "MERGE_BASE=${MERGE_BASE}" >> "$GITHUB_ENV"

      # If possible, run the "abridged" jobs on PRs to save time.
      #
      # The goal is to still cover all persistence code paths:
      #   1. SQL
      #   2. NoSQL (ie Cassandra)
      #   3. Elasticsearch/OpenSearch
      # All other jobs for the remaining databases only run a small set of smoke tests.
      #
      # Exceptions to this optimization:
      #   1. commit is not a PR
      #   2. `test-all-dbs` label set on PR
      #   3. changes to persistence packages were made
      - name: Determine test scope
        id: determine_scope
        run: |
          FULL_TEST_REASON=""

          event_name="${{ github.event_name }}"
          labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
          merge_base="${MERGE_BASE:-}"

          echo "event_name=$event_name"
          echo "labels=$labels"
          echo "merge_base=$merge_base"

          # Push events (main, release branches) run all tests on all DBs
          if [[ "$event_name" == "push" ]]; then
            FULL_TEST_REASON="Running full tests on all DBs (push event)."
          # Check for test-all-dbs label
          elif echo "$labels" | jq -e 'any(. == "test-all-dbs")' > /dev/null 2>&1; then
            FULL_TEST_REASON="Running full tests on all DBs (test-all-dbs label)."
          # Check for persistence code changes
          elif [[ -n "$merge_base" ]]; then
            echo "Changed persistence files:"
            git diff --name-only "$merge_base" "$COMMIT" | grep -E "^(common/persistence/|schema/)" || echo "(none)"
            if git diff --name-only "$merge_base" "$COMMIT" | grep -qE "^(common/persistence/|schema/)"; then
              FULL_TEST_REASON="Running full tests on all DBs (persistence code changes)."
            fi
          fi

          echo "full_test_reason=$FULL_TEST_REASON" >> "$GITHUB_OUTPUT"

      - name: ${{ steps.determine_scope.outputs.full_test_reason && 'ℹ️ Full tests' || 'ℹ️ Smoke tests' }}
        run: echo "::notice::${{ steps.determine_scope.outputs.full_test_reason || 'Running smoke tests on extended DBs. Add the test-all-dbs label to run all tests on all DBs.' }}"

      # Primary DBs always get full tests, extended DBs get smoke tests (1 job) in abridged PRs.
      - name: Build job matrix
        id: build_matrix
        env:
          FULL_TEST_REASON: ${{ steps.determine_scope.outputs.full_test_reason }}
          DB_CONFIGS: |
            cass_es:
              persistence_type: nosql
              persistence_driver: cassandra
              containers: [cassandra, elasticsearch]
              required: true
            cass_es8:
              persistence_type: nosql
              persistence_driver: cassandra
              containers: [cassandra, elasticsearch8]
            cass_os2:
              persistence_type: nosql
              persistence_driver: cassandra
              containers: [cassandra, opensearch2]
            cass_os3:
              persistence_type: nosql
              persistence_driver: cassandra
              containers: [cassandra, opensearch3]
            sqlite:
              persistence_type: sql
              persistence_driver: sqlite
              containers: []
            mysql8:
              persistence_type: sql
              persistence_driver: mysql8
              containers: [mysql]
            postgres12:
              persistence_type: sql
              persistence_driver: postgres12
              containers: [postgresql]
              required: true
            postgres12_pgx:
              persistence_type: sql
              persistence_driver: postgres12_pgx
              containers: [postgresql]
          JOB_TYPES: |
            functest:
              cmd: make functional-test-coverage
              test_timeout: 35m
              github_timeout: 40
              sharded: true
            smoke:
              cmd: make functional-test-coverage
              test_timeout: 5m
              github_timeout: 10
              test_args: '"-run=TestActivityTestSuite|TestSignalWorkflowTestSuite|TestWorkflowTestSuite"'
            ndc:
              cmd: make functional-test-ndc-coverage
              test_timeout: 10m
              github_timeout: 15
            xdc:
              cmd: make functional-test-xdc-coverage
              test_timeout: 30m
              github_timeout: 35
        run: |
          # Convert YAML inputs to JSON for jq.
          DBS_JSON=$(yq -o=json <<< "$DB_CONFIGS")
          JOBS_JSON=$(yq -o=json <<< "$JOB_TYPES")

          # Build the job matrix as a JSON array.
          MATRIX=$(
            jq -c -n \
              --argjson dbs "$DBS_JSON" \
              --argjson jobs "$JOBS_JSON" \
              --argjson shard_count "$SHARD_COUNT" \
              --arg full_test_reason "$FULL_TEST_REASON" \
            '
            [
              $dbs | to_entries[] as $db |

              # Full test suite for required DBs (or all DBs when full_test_reason is set).
              (if $db.value.required or ($full_test_reason != "") then
                $jobs | to_entries[] | select(.key != "smoke")
              else
                $jobs | to_entries[] | select(.key == "smoke")
              end) as $job |

              # Base entry: merge db config + job config + name
              ($db.value + ($job.value | del(.sharded)) + {name: $db.key}) as $base |

              # If sharded, emit one entry per shard; otherwise emit one entry
              if $job.value.sharded then
                range($shard_count) as $i |
                $base + {
                  display_name: "shard\($i)",
                  shard_index: $i,
                  total_shards: $shard_count
                }
              else
                $base + {display_name: $job.key}
              end
            ]
            '
          )

          echo "job_matrix=$MATRIX" >> "$GITHUB_OUTPUT"
          echo "Generated $(jq length <<< "$MATRIX") jobs"

  pre-build:
    name: Pre-build for cache
    runs-on: ubuntu-24.04-arm # NOTE: avoid using larger runners as they have high provisioning time (up to several minutes)
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ env.COMMIT }}

      - name: Prepare Go
        id: prepare-go
        uses: ./.github/actions/prepare-go

      - run: make pre-build-functional-test-coverage

      - name: Save Go dependencies
        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        if: ${{ steps.prepare-go.outputs.dependency_cache_hit != 'true' }}
        with:
          path: ${{ steps.prepare-go.outputs.dependency_cache_path }}
          key: ${{ steps.prepare-go.outputs.dependency_cache_primary_key }}

      - name: Save Go tools
        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        if: ${{ steps.prepare-go.outputs.tools_cache_hit != 'true' }}
        with:
          path: ${{ steps.prepare-go.outputs.tools_cache_path }}
          key: ${{ steps.prepare-go.outputs.tools_cache_primary_key }}

      - name: Save Go build outputs
        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        if: ${{ steps.prepare-go.outputs.build_cache_hit != 'true' }}
        with:
          path: ${{ steps.prepare-go.outputs.build_cache_path }}
          key: ${{ steps.prepare-go.outputs.build_cache_primary_key }}

  misc-checks:
    name: Misc checks
    needs: [pre-build]
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ env.COMMIT }}
          # buf-breaking tries to compare HEAD against merge base so we need to be able to find it
          fetch-depth: 100

      - name: Prepare Go
        uses: ./.github/actions/prepare-go

      - uses: jdx/mise-action@dba19683ed58901619b14f395a24841710cb4925 # v4.1.0
        with:
          working_directory: .github
          install_args: protoc

      - run: GOOS=windows GOARCH=amd64 make clean-bins bins

      - run: GOOS=darwin GOARCH=arm64 make clean-bins bins

      - run: make clean-bins ci-build-misc

  unit-test:
    name: Unit test
    needs: [pre-build]
    runs-on: ubuntu-24.04-arm
    env:
      TEST_TIMEOUT: 15m
      GITHUB_TIMEOUT: 20
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ env.COMMIT }}

      - name: Prepare Go
        uses: ./.github/actions/prepare-go

      - name: Run unit tests
        timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }}
        run: |
          export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m"
          ./develop/github/monitor_test.sh make unit-test-coverage

      - name: Post-test reporting
        if: always()
        uses: ./.github/actions/post-test-reporting
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          job_name: Unit test
          artifact_name_suffix: unit-test

  leak-test:
    name: Resource leak test
    needs: [pre-build]
    runs-on: ubuntu-24.04-arm
    env:
      GITHUB_TIMEOUT: 10
      LEAK_OUTPUT_DIR: ${{ github.workspace }}/.testoutput/leakcheck
      LEAK_GC_SETTLE_TIMEOUT: 30s
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ env.COMMIT }}

      - name: Prepare Go
        uses: ./.github/actions/prepare-go

      - name: Run resource leak test
        timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }}
        run: make LEAK_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m" leak-test

      - name: Upload diagnostics
        if: failure()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: leak-diagnostics
          path: ${{ env.LEAK_OUTPUT_DIR }}
          if-no-files-found: ignore

  integration-test:
    name: Integration test
    needs: [pre-build]
    runs-on: ubuntu-24.04-arm
    env:
      CONTAINERS: cassandra mysql postgresql elasticsearch
      TEST_TIMEOUT: 10m
      GITHUB_TIMEOUT: 15
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ env.COMMIT }}

      - name: Start containerized dependencies
        uses: ./.github/actions/docker-compose-start
        with:
          compose_file: ${{ env.DOCKER_COMPOSE_FILE }}
          services: ${{ env.CONTAINERS }}

      - name: Prepare Go
        uses: ./.github/actions/prepare-go

      - name: Wait for containerized dependencies to be healthy
        uses: ./.github/actions/docker-compose-healthy
        with:
          compose_file: ${{ env.DOCKER_COMPOSE_FILE }}
          services: ${{ env.CONTAINERS }}

      - name: Run integration test
        timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }}
        run: |
          export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m"
          ./develop/github/monitor_test.sh make integration-test-coverage

      - name: Post-test reporting
        if: always()
        uses: ./.github/actions/post-test-reporting
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          job_name: Integration test
          artifact_name_suffix: integration-test

      - name: Stop containerized dependencies
        if: ${{ always() }}
        uses: ./.github/actions/docker-compose-stop
        with:
          compose_file: ${{ env.DOCKER_COMPOSE_FILE }}
          flags: -v

  # Root job name includes matrix details so it is unique per job variant.
  # This MUST stay in sync with the `job_name` passed to the job-id action below.
  functional-test:
    # Display name shown in the UI. The job-id lookup uses this exact value.
    name: Functional test (${{ matrix.name }}, ${{ matrix.display_name }})
    needs: [pre-build, test-setup]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.test-setup.outputs.job_matrix) }}
    runs-on: ubuntu-24.04-arm
    env:
      PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
      PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
      TEST_TIMEOUT: ${{ matrix.test_timeout }}
      GITHUB_TIMEOUT: ${{ matrix.github_timeout }}
      CONTAINERS: ${{ join(matrix.containers, ' ') }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ env.COMMIT }}

      - name: Start containerized dependencies
        uses: ./.github/actions/docker-compose-start
        with:
          compose_file: ${{ env.DOCKER_COMPOSE_FILE }}
          services: ${{ env.CONTAINERS }}
          cache: true

      - name: Prepare Go
        uses: ./.github/actions/prepare-go

      - name: ${{ matrix.display_name == 'smoke' && 'ℹ️ Smoke test' || 'ℹ️ Full test' }}
        run: echo "::notice::${{ matrix.display_name == 'smoke' && 'This is a smoke test. Add the test-all-dbs label to run all tests on all DBs.' || needs.test-setup.outputs.full_test_reason }}"

      - name: Wait for containerized dependencies to be healthy
        uses: ./.github/actions/docker-compose-healthy
        with:
          compose_file: ${{ env.DOCKER_COMPOSE_FILE }}
          services: ${{ env.CONTAINERS }}

      - name: Run functional test
        timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }}
        run: |
          export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m"
          ./develop/github/monitor_test.sh ${{ matrix.cmd }}
        env:
          TEST_TOTAL_SHARDS: ${{ matrix.total_shards }}
          TEST_SHARD_INDEX: ${{ matrix.total_shards && matrix.shard_index }} # guard with total_shards to avoid falsy eval of shard_index=0
          TEST_ARGS: "${{ matrix.test_args }}"
          TEMPORAL_TEST_LOG_FILE: ${{ github.workspace }}/.testoutput/debug.log
          TEMPORAL_TEST_LOG_LEVEL: warn
          TEMPORAL_TEST_CLUSTER_EVENTS_FILE: ${{ github.workspace }}/.testoutput/test-cluster-events.jsonl

      - name: Dump container logs
        if: ${{ failure() && toJson(matrix.containers) != '[]' }}
        run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} logs --no-color

      - name: Post-test reporting
        if: always()
        uses: ./.github/actions/post-test-reporting
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          job_name: Functional test (${{ matrix.name }}, ${{ matrix.display_name }})
          artifact_name_suffix: ${{ matrix.name }}--${{ matrix.display_name }}--functional-test
          codecov_flag: functional-test
          debug_log_path: ${{ github.workspace }}/.testoutput/debug.log

      - name: Stop containerized dependencies
        if: always()
        uses: ./.github/actions/docker-compose-stop
        with:
          compose_file: ${{ env.DOCKER_COMPOSE_FILE }}
          flags: -v

  mixed-brain-test:
    name: Mixed brain test
    needs: [pre-build]
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ env.COMMIT }}

      - name: Start PostgreSQL
        uses: ./.github/actions/docker-compose-start
        with:
          compose_file: ${{ env.DOCKER_COMPOSE_FILE }}
          services: postgresql

      - name: Prepare Go
        uses: ./.github/actions/prepare-go

      - name: Install PostgreSQL schema
        run: make install-schema-postgresql12

      - name: Run mixed brain test
        timeout-minutes: 20
        run: ./develop/github/monitor_test.sh make mixed-brain-test
        env:
          TEST_TIMEOUT: 18m
          MIXED_BRAIN_TEST_DURATION: 5m
          PERSISTENCE_DRIVER: postgres12

      - name: Print memory snapshot
        if: always()
        run: cat .testoutput/memory/memory-snapshot.txt || true

      - name: Print current server logs
        if: always()
        run: cat .testoutput/mixedbrain_process-current.log || true

      - name: Print release server logs
        if: always()
        run: cat .testoutput/mixedbrain_process-release.log || true

      - name: Print Omes logs
        if: always()
        run: cat .testoutput/mixedbrain_omes_*.log || true

      - name: Stop containerized dependencies
        if: always()
        uses: ./.github/actions/docker-compose-stop
        with:
          compose_file: ${{ env.DOCKER_COMPOSE_FILE }}
          flags: -v

  test-status:
    if: always()
    name: Test Status
    needs:
      - misc-checks
      - unit-test
      - integration-test
      - functional-test
    runs-on: ubuntu-latest
    env:
      RESULTS: ${{ toJSON(needs.*.result) }}
    steps:
      - name: Check results
        run: |
          # all statuses must be success
          if [[ -n $(echo "$RESULTS" | jq '.[] | select (. != "success")') ]]; then
            exit 1
          fi

  notify-failure:
    name: Notify Slack on Failure
    if: |
      always() &&
      github.ref == 'refs/heads/main' &&
      needs.test-status.result == 'failure'
    needs: test-status
    runs-on: ubuntu-latest
    permissions:
      contents: read
      actions: read
    steps:
      - name: Checkout code
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

      - name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version-file: "go.mod"
          cache: true

      - name: Send Slack notification
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          GH_TOKEN: ${{ github.token }}
        run: |
          go run ./cmd/tools/ci-notify alert \
            --run-id "${{ github.run_id }}" \
            --slack-webhook "$SLACK_WEBHOOK"

  notify-data-race:
    name: Notify Slack on Data Race
    # Runs on both passing and failing main runs: the race detector can trip on
    # one attempt and pass on a retry, leaving the run green while a data race
    # was still recorded in the test-summary artifacts.
    if: |
      always() &&
      github.ref == 'refs/heads/main' &&
      (needs.test-status.result == 'success' || needs.test-status.result == 'failure')
    needs: test-status
    runs-on: ubuntu-latest
    permissions:
      contents: read
      actions: read
    steps:
      - name: Checkout code
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

      - name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version-file: "go.mod"
          cache: true

      - name: Send Slack notification
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          GH_TOKEN: ${{ github.token }}
        run: |
          go run ./cmd/tools/ci-notify data-race \
            --run-id "${{ github.run_id }}" \
            --slack-webhook "$SLACK_WEBHOOK"