mirror of
https://github.com/temporalio/temporal.git
synced 2026-08-30 18:41:49 -07:00
80a957acec
## What changed? Collecting OTEL traces from functional tests. And packaging them up as an artifact. ## Why Help debug flaky tests. ## Testing You can see the trace artifacts on [this run](https://github.com/temporalio/temporal/actions/runs/15719131802?pr=7871).
894 lines
35 KiB
YAML
894 lines
35 KiB
YAML
name: All Tests
|
|
run-name: ${{ inputs.test_name }}${{ inputs.test_dbs }}
|
|
|
|
on:
|
|
pull_request:
|
|
push:
|
|
branches:
|
|
- main
|
|
- release/**
|
|
- cloud/**
|
|
|
|
workflow_dispatch:
|
|
inputs:
|
|
commit:
|
|
description: "Commit SHA"
|
|
required: true
|
|
run_single_functional_test:
|
|
description: "Run a single functional test"
|
|
type: boolean
|
|
default: false
|
|
run_single_unit_test:
|
|
description: "Run a single unit test (INSTEAD of functional test)"
|
|
type: boolean
|
|
default: false
|
|
unit_test_directory:
|
|
description: "[Unit Test Only] Directory to run unit tests in"
|
|
type: string
|
|
default: "./temporal"
|
|
n_runs:
|
|
description: "Number of times to repeat the test per database type (except for unit tests, start with n=25 or lower to avoid OOMKill)"
|
|
type: number
|
|
default: 1
|
|
test_name:
|
|
description: "Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
|
|
type: string
|
|
timeout_minutes:
|
|
description: "Test timeout in minutes"
|
|
type: number
|
|
default: 120
|
|
test_runner:
|
|
description: "Which runner to use. Choose higher RAM if your n_runs is high."
|
|
type: choice
|
|
default: "16GB RAM (ubuntu-latest)"
|
|
options:
|
|
- "16GB RAM (ubuntu-latest)"
|
|
- "64GB RAM (ubuntu-latest-16-cores)"
|
|
test_dbs:
|
|
description: 'List of DBs to test on (i.e. ["sqlite", "cassandra", "mysql8", "postgres12"])'
|
|
type: string
|
|
default: '["sqlite"]'
|
|
|
|
concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
|
|
group: run-tests-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
# For workflow_dispatch: use the given commit.
|
|
# For pull_request: use the head of the PR branch (not the merge branch which is the default!)
|
|
# For push: use the pushed commit.
|
|
COMMIT: ${{ github.event.inputs.commit || github.event.pull_request.head.sha || github.sha }}
|
|
PR_BASE_COMMIT: ${{ github.event.pull_request.base.sha }}
|
|
DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
|
|
TEMPORAL_VERSION_CHECK_DISABLED: 1
|
|
|
|
jobs:
|
|
set-up-single-test:
|
|
name: Set up single test
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
|
|
total_shards: ${{ steps.generate_output.outputs.shards }}
|
|
github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
|
|
test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
|
|
single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
|
|
runs_on: ${{ steps.generate_output.outputs.runs_on }}
|
|
dbs: ${{ inputs.test_dbs }}
|
|
modified_unit_test_suites: ${{ env.modified_unit_test_suites }}
|
|
modified_integration_test_suites: ${{ env.modified_integration_test_suites }}
|
|
modified_functional_test_suites: ${{ env.modified_functional_test_suites }}
|
|
modified_functional_ndc_test_suites: ${{ env.modified_functional_ndc_test_suites }}
|
|
modified_functional_xdc_test_suites: ${{ env.modified_functional_xdc_test_suites }}
|
|
steps:
|
|
- id: generate_output
|
|
run: |
|
|
shards=3
|
|
timeout=35 # update this to TEST_TIMEOUT if you update the Makefile
|
|
runs_on='["ubuntu-latest"]'
|
|
if [[ "${{ inputs.run_single_functional_test }}" == "true" || "${{ inputs.run_single_unit_test }}" == "true" ]]; then
|
|
shards=1
|
|
timeout=${{ inputs.timeout_minutes }}
|
|
single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
|
|
if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-latest-16-cores)" ]]; then
|
|
runs_on='[ "ubuntu-latest-16-cores" ]'
|
|
fi
|
|
fi
|
|
{
|
|
echo "shard_indices=[ $(seq -s, 0 $((shards-1))) ]"
|
|
echo "shards=$shards"
|
|
echo "github_timeout=$((timeout+5))"
|
|
echo "test_timeout=${timeout}m"
|
|
echo "single_test_args=$single_test_args"
|
|
echo "runs_on=$runs_on"
|
|
} >> "$GITHUB_OUTPUT"
|
|
- id: cat_output
|
|
run: |
|
|
cat "$GITHUB_OUTPUT"
|
|
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
ref: ${{ env.COMMIT }}
|
|
fetch-depth: 0
|
|
|
|
- name: Fetch base branch
|
|
run: git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}
|
|
|
|
- name: Compute merge base
|
|
if: ${{ github.event_name == 'pull_request' }}
|
|
run: |
|
|
MERGE_BASE="$(git merge-base "${{ env.COMMIT }}" "${{ github.event.pull_request.base.ref }}")"
|
|
echo "MERGE_BASE=${MERGE_BASE}" >> "$GITHUB_ENV"
|
|
|
|
set -exuo pipefail
|
|
|
|
go run ./cmd/tools/test/find_altered_tests.go \
|
|
-c unit \
|
|
-c integration \
|
|
-c functional \
|
|
-c functional_ndc \
|
|
-c functional_xdc \
|
|
-s "${MERGE_BASE}" \
|
|
-t "${COMMIT}" | tee -a "$GITHUB_ENV"
|
|
shell: bash
|
|
|
|
pre-build:
|
|
name: Pre-build for cache
|
|
strategy:
|
|
fail-fast: false
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
ref: ${{ env.COMMIT }}
|
|
|
|
- uses: actions/setup-go@v5
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
with:
|
|
go-version-file: "go.mod"
|
|
cache: false # do our own caching
|
|
|
|
- name: Restore dependencies
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
id: restore-deps
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/go/pkg/mod
|
|
key: go-${{ runner.os }}${{ runner.arch }}-deps-${{ hashFiles('go.sum') }}
|
|
|
|
- run: make pre-build-functional-test-coverage
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
|
|
- name: Save dependencies
|
|
uses: actions/cache/save@v4
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test && steps.restore-deps.outputs.cache-hit != 'true' }}
|
|
with:
|
|
path: ~/go/pkg/mod
|
|
key: ${{ steps.restore-deps.outputs.cache-primary-key }}
|
|
|
|
- name: Save build outputs
|
|
uses: actions/cache/save@v4
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
with:
|
|
path: ~/.cache/go-build
|
|
key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
|
|
|
|
misc-checks:
|
|
name: Misc checks
|
|
needs: pre-build
|
|
strategy:
|
|
fail-fast: false
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
ref: ${{ env.COMMIT }}
|
|
# buf-breaking tries to compare HEAD against merge base so we need to be able to find it
|
|
fetch-depth: 100
|
|
|
|
- uses: actions/setup-go@v5
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
with:
|
|
go-version-file: "go.mod"
|
|
cache: false # do our own caching
|
|
|
|
- name: Restore dependencies
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/go/pkg/mod
|
|
key: go-${{ runner.os }}${{ runner.arch }}-deps-${{ hashFiles('go.sum') }}
|
|
|
|
- name: Restore build outputs
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/.cache/go-build
|
|
key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
|
|
|
|
- uses: arduino/setup-protoc@v3
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
with:
|
|
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- run: GOOS=windows GOARCH=amd64 make clean-bins bins
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
|
|
- run: GOOS=darwin GOARCH=arm64 make clean-bins bins
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
|
|
- run: make clean-bins ci-build-misc
|
|
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
|
|
|
|
unit-test:
|
|
if: ${{ inputs.run_single_functional_test != true }}
|
|
name: Unit test
|
|
needs: [pre-build, set-up-single-test]
|
|
strategy:
|
|
fail-fast: false
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
ref: ${{ env.COMMIT }}
|
|
|
|
- uses: actions/setup-go@v5
|
|
with:
|
|
go-version-file: "go.mod"
|
|
cache: false # do our own caching
|
|
|
|
- name: Restore dependencies
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/go/pkg/mod
|
|
key: go-${{ runner.os }}${{ runner.arch }}-deps-${{ hashFiles('go.sum') }}
|
|
|
|
- name: Restore build outputs
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/.cache/go-build
|
|
key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
|
|
|
|
- name: Run unit tests
|
|
timeout-minutes: 15
|
|
run: make unit-test-coverage
|
|
env:
|
|
UNIT_TEST_DIRS: ${{ inputs.unit_test_directory }}
|
|
TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
|
|
TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
|
|
|
|
- name: Generate test summary
|
|
uses: mikepenz/action-junit-report@v5.0.0-rc01
|
|
if: failure()
|
|
with:
|
|
report_paths: ./.testoutput/junit.*.xml
|
|
detailed_summary: true
|
|
check_annotations: false
|
|
annotate_only: true
|
|
skip_annotations: true
|
|
|
|
- name: Upload code coverage to Codecov
|
|
uses: codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: unit-test
|
|
|
|
- name: Upload test results to Codecov
|
|
if: ${{ !cancelled() }}
|
|
uses: codecov/test-results-action@v1
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: unit-test
|
|
|
|
- name: Upload test results to GitHub
|
|
# Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
|
|
uses: actions/upload-artifact@v4.4.3
|
|
if: ${{ !cancelled() && !inputs.run_single_unit_test }}
|
|
with:
|
|
name: junit-xml--${{github.run_id}}--${{github.run_attempt}}--unit-test
|
|
path: .testoutput
|
|
include-hidden-files: true
|
|
retention-days: 28
|
|
|
|
# Ensure this doesn't contribute to the junit output.
|
|
- name: Flaky Unit Test Detection
|
|
if: ${{ !cancelled() && !inputs.run_single_unit_test && env.MODIFIED_TEST_SUITES != '' }}
|
|
timeout-minutes: 30
|
|
run: |
|
|
echo "Detecting flaky unit tests: ${{ needs.set-up-single-test.outputs.modified_unit_test_suites }}"
|
|
make unit-test
|
|
env:
|
|
FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
|
|
TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_unit_test_suites }} -count=10"
|
|
TEST_TIMEOUT: 35
|
|
|
|
integration-test:
|
|
if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
|
|
name: Integration test
|
|
needs: [pre-build, set-up-single-test]
|
|
strategy:
|
|
fail-fast: false
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
ref: ${{ env.COMMIT }}
|
|
|
|
- name: Start containerized dependencies
|
|
uses: hoverkraft-tech/compose-action@v2.0.1
|
|
with:
|
|
compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
|
|
services: |
|
|
cassandra
|
|
mysql
|
|
postgresql
|
|
down-flags: -v
|
|
|
|
- uses: actions/setup-go@v5
|
|
with:
|
|
go-version-file: "go.mod"
|
|
cache: false # do our own caching
|
|
|
|
- name: Restore dependencies
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/go/pkg/mod
|
|
key: go-${{ runner.os }}${{ runner.arch }}-deps-${{ hashFiles('go.sum') }}
|
|
|
|
- name: Restore build outputs
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/.cache/go-build
|
|
key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
|
|
|
|
- name: Run integration test
|
|
timeout-minutes: 15
|
|
run: make integration-test-coverage
|
|
|
|
- name: Generate test summary
|
|
uses: mikepenz/action-junit-report@v5.0.0-rc01
|
|
if: failure()
|
|
with:
|
|
report_paths: ./.testoutput/junit.*.xml
|
|
detailed_summary: true
|
|
check_annotations: false
|
|
annotate_only: true
|
|
skip_annotations: true
|
|
|
|
- name: Upload code coverage to Codecov
|
|
uses: codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: integration-test
|
|
|
|
- name: Upload test results to Codecov
|
|
if: ${{ !cancelled() }}
|
|
uses: codecov/test-results-action@v1
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: integration-test
|
|
|
|
- name: Upload test results to GitHub
|
|
# Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
|
|
uses: actions/upload-artifact@v4.4.3
|
|
if: ${{ !cancelled() }}
|
|
with:
|
|
name: junit-xml--${{github.run_id}}--${{github.run_attempt}}--integration-test
|
|
path: .testoutput
|
|
include-hidden-files: true
|
|
retention-days: 28
|
|
|
|
# Ensure this doesn't contribute to the junit output.
|
|
- name: Flaky Integration Test Detection
|
|
if: ${{ env.MODIFIED_TEST_SUITES != '' }}
|
|
timeout-minutes: 30
|
|
run: |
|
|
echo "Detecting flaky integration tests: ${{ needs.set-up-single-test.outputs.modified_integration_test_suites }}"
|
|
make integration-test-coverage
|
|
env:
|
|
FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
|
|
TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_integration_test_suites }} -count=5"
|
|
|
|
- name: Tear down docker compose
|
|
if: ${{ always() }}
|
|
run: |
|
|
docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
|
|
|
|
functional-test:
|
|
if: ${{ inputs.run_single_unit_test != true }}
|
|
name: Functional test
|
|
needs: [pre-build, set-up-single-test]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
runs-on: ${{ fromJson(needs.set-up-single-test.outputs.runs_on) }}
|
|
shard_index: ${{ fromJson(needs.set-up-single-test.outputs.shard_indices) }}
|
|
name:
|
|
- cass_es
|
|
- cass_es8
|
|
- sqlite
|
|
- mysql8
|
|
- postgres12
|
|
- postgres12_pgx
|
|
include:
|
|
- name: cass_es
|
|
persistence_type: nosql
|
|
persistence_driver: cassandra
|
|
containers: [cassandra, elasticsearch]
|
|
es_version: v7
|
|
# Cassandra tests need a larger instance
|
|
runs_on_override: ubuntu-latest-8-cores
|
|
- name: cass_es8
|
|
persistence_type: nosql
|
|
persistence_driver: cassandra
|
|
containers: [cassandra, elasticsearch8]
|
|
es_version: v8
|
|
# Cassandra tests need a larger instance
|
|
runs_on_override: ubuntu-latest-8-cores
|
|
- name: sqlite
|
|
persistence_type: sql
|
|
persistence_driver: sqlite
|
|
containers: []
|
|
- name: mysql8
|
|
persistence_type: sql
|
|
persistence_driver: mysql8
|
|
containers: [mysql]
|
|
- name: postgres12
|
|
persistence_type: sql
|
|
persistence_driver: postgres12
|
|
containers: [postgresql]
|
|
- name: postgres12_pgx
|
|
persistence_type: sql
|
|
persistence_driver: postgres12_pgx
|
|
containers: [postgresql]
|
|
runs-on: ${{ matrix.runs_on_override || matrix.runs-on }}
|
|
env:
|
|
TEST_TOTAL_SHARDS: ${{ needs.set-up-single-test.outputs.total_shards }}
|
|
TEST_SHARD_INDEX: ${{ matrix.shard_index }}
|
|
PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
|
|
PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
|
|
TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
|
|
steps:
|
|
- uses: ScribeMD/docker-cache@0.3.7
|
|
if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
|
|
with:
|
|
key: docker-${{ runner.os }}${{ runner.arch }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
|
|
|
|
- uses: actions/checkout@v4
|
|
if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
ref: ${{ env.COMMIT }}
|
|
|
|
- name: Start containerized dependencies
|
|
if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
|
|
uses: hoverkraft-tech/compose-action@v2.0.1
|
|
with:
|
|
compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
|
|
services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}"
|
|
down-flags: -v
|
|
|
|
- uses: actions/setup-go@v5
|
|
if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
|
|
with:
|
|
go-version-file: "go.mod"
|
|
cache: false # do our own caching
|
|
|
|
- name: Restore dependencies
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/go/pkg/mod
|
|
key: go-${{ runner.os }}${{ runner.arch }}-deps-${{ hashFiles('go.sum') }}
|
|
|
|
- name: Restore build outputs
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/.cache/go-build
|
|
key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
|
|
|
|
- name: Get job ID from workflow run
|
|
id: get_job_id
|
|
run: |
|
|
set -eu
|
|
test_job_name="Functional test (${{ matrix.runs_on_override || matrix.runs-on }}, ${{matrix.shard_index}}, ${{matrix.name}})"
|
|
job_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100"
|
|
# curl or jq might silently fail, but we are okay with that.
|
|
# We don't want to fail the job if we can't get the job id.
|
|
job_id=$(curl -L \
|
|
-H "Accept: application/vnd.github+json" \
|
|
-H "Authorization: Bearer ${{secrets.GITHUB_TOKEN}}" \
|
|
-H "X-GitHub-Api-Version: 2022-11-28" \
|
|
"$job_url" \
|
|
| jq "first(.jobs[] | select(.name == \"$test_job_name\") | .id)")
|
|
echo "Job id is $job_id"
|
|
echo "JOB_ID=${job_id:-unknown}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Run functional test
|
|
if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
|
|
timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
|
|
run: make OTEL=true functional-test-coverage
|
|
env:
|
|
TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
|
|
|
|
- name: Generate test summary
|
|
uses: mikepenz/action-junit-report@v5.0.0-rc01
|
|
if: failure()
|
|
with:
|
|
report_paths: ./.testoutput/junit.*.xml
|
|
detailed_summary: true
|
|
check_annotations: false
|
|
annotate_only: true
|
|
skip_annotations: true
|
|
|
|
- name: Upload code coverage to Codecov
|
|
uses: codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: functional-test
|
|
|
|
- name: Upload test results to Codecov
|
|
if: ${{ !cancelled() }}
|
|
uses: codecov/test-results-action@v1
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: functional-test
|
|
|
|
- name: Upload test results to GitHub
|
|
# Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
|
|
uses: actions/upload-artifact@v4.4.3
|
|
if: ${{ !cancelled() && !inputs.run_single_functional_test }}
|
|
with:
|
|
name: junit-xml--${{github.run_id}}--${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{ matrix.runs_on_override || matrix.runs-on }}--${{matrix.name}}--${{matrix.shard_index}}--functional-test
|
|
path: .testoutput
|
|
include-hidden-files: true
|
|
retention-days: 28
|
|
|
|
# Ensure this doesn't contribute to the junit output.
|
|
- name: Flaky Functional Test Detection
|
|
if: ${{ (env.MODIFIED_TEST_SUITES != '') && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
|
|
timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
|
|
run: |
|
|
echo "Detecting flaky functional tests: ${{ needs.set-up-single-test.outputs.modified_integration_test_suites }}"
|
|
make functional-test-coverage
|
|
env:
|
|
FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
|
|
TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_test_suites }} -count=5"
|
|
|
|
# Upload OpenTelemetry traces.
|
|
- name: Upload OpenTelemetry traces
|
|
if: always() && github.ref == 'refs/heads/main'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{matrix.shard_index}}--${{github.run_attempt}}--${{matrix.name}}--functional-test
|
|
path: /tmp/traces.json
|
|
|
|
functional-test-xdc:
|
|
if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
|
|
name: Functional test xdc
|
|
needs: [pre-build, set-up-single-test]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
name: [cass_es, cass_es8, mysql8, postgres12, postgres12_pgx]
|
|
include:
|
|
- name: cass_es
|
|
persistence_type: nosql
|
|
persistence_driver: elasticsearch
|
|
parallel_flags: ""
|
|
containers: [cassandra, elasticsearch]
|
|
# Cassandra tests need a larger instance
|
|
runs_on_override: ubuntu-latest-8-cores
|
|
- name: cass_es8
|
|
persistence_type: nosql
|
|
persistence_driver: elasticsearch
|
|
parallel_flags: ""
|
|
containers: [cassandra, elasticsearch8]
|
|
# Cassandra tests need a larger instance
|
|
runs_on_override: ubuntu-latest-8-cores
|
|
- name: mysql8
|
|
persistence_type: sql
|
|
persistence_driver: mysql8
|
|
parallel_flags: ""
|
|
containers: [mysql]
|
|
- name: postgres12
|
|
persistence_type: sql
|
|
persistence_driver: postgres12
|
|
parallel_flags: "-parallel=2" # reduce parallelism for postgres
|
|
containers: [postgresql]
|
|
- name: postgres12_pgx
|
|
persistence_type: sql
|
|
persistence_driver: postgres12_pgx
|
|
parallel_flags: "-parallel=2" # reduce parallelism for postgres
|
|
containers: [postgresql]
|
|
runs-on: ${{ matrix.runs_on_override || 'ubuntu-latest' }}
|
|
env:
|
|
PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
|
|
PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
|
|
TEST_PARALLEL_FLAGS: ${{ matrix.parallel_flags }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
ref: ${{ env.COMMIT }}
|
|
|
|
- name: Start containerized dependencies
|
|
if: ${{ toJson(matrix.containers) != '[]' }}
|
|
uses: hoverkraft-tech/compose-action@v2.0.1
|
|
with:
|
|
compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
|
|
services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}"
|
|
down-flags: -v
|
|
|
|
- uses: actions/setup-go@v5
|
|
with:
|
|
go-version-file: "go.mod"
|
|
cache: false # do our own caching
|
|
|
|
- name: Restore dependencies
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/go/pkg/mod
|
|
key: go-${{ runner.os }}${{ runner.arch }}-deps-${{ hashFiles('go.sum') }}
|
|
|
|
- name: Restore build outputs
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/.cache/go-build
|
|
key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
|
|
|
|
- name: Get job ID from workflow run
|
|
id: get_job_id
|
|
run: |
|
|
set -eu
|
|
test_job_name="Functional test xdc (${{matrix.name}})"
|
|
job_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100"
|
|
# curl or jq might silently fail, but we are okay with that.
|
|
# We don't want to fail the job if we can't get the job id.
|
|
job_id=$(curl -L \
|
|
-H "Accept: application/vnd.github+json" \
|
|
-H "Authorization: Bearer ${{secrets.GITHUB_TOKEN}}" \
|
|
-H "X-GitHub-Api-Version: 2022-11-28" \
|
|
"$job_url" \
|
|
| jq "first(.jobs[] | select(.name == \"$test_job_name\") | .id)")
|
|
echo "Job id is $job_id"
|
|
echo "JOB_ID=${job_id-unknown}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Run functional test xdc
|
|
timeout-minutes: 35 # update this to TEST_TIMEOUT+5 if you update the Makefile
|
|
run: make OTEL=true functional-test-xdc-coverage
|
|
|
|
- name: Generate test summary
|
|
uses: mikepenz/action-junit-report@v5.0.0-rc01
|
|
if: failure()
|
|
with:
|
|
report_paths: ./.testoutput/junit.*.xml
|
|
detailed_summary: true
|
|
check_annotations: false
|
|
annotate_only: true
|
|
skip_annotations: true
|
|
|
|
- name: Upload code coverage to Codecov
|
|
uses: codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: functional-test-xdc
|
|
|
|
- name: Upload test results to Codecov
|
|
if: ${{ !cancelled() }}
|
|
uses: codecov/test-results-action@v1
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: functional-test-xdc
|
|
|
|
- name: Upload test results to GitHub
|
|
# Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
|
|
uses: actions/upload-artifact@v4.4.3
|
|
if: ${{ !cancelled() }}
|
|
with:
|
|
name: junit-xml--${{github.run_id}}--${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{matrix.name}}--functional-test-xdc
|
|
path: .testoutput
|
|
include-hidden-files: true
|
|
retention-days: 28
|
|
|
|
# Ensure this doesn't contribute to the junit output.
|
|
- name: Flaky Functional XDC Test Detection
|
|
if: ${{ env.MODIFIED_TEST_SUITES != '' }}
|
|
timeout-minutes: 60
|
|
run: |
|
|
echo "Detecting flaky functional xdc tests: ${{ needs.set-up-single-test.outputs.modified_functional_xdc_test_suites }}"
|
|
make functional-test-xdc-coverage
|
|
env:
|
|
FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
|
|
TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_xdc_test_suites }} -count=5"
|
|
|
|
# Upload OpenTelemetry traces.
|
|
- name: Upload OpenTelemetry traces
|
|
if: always() && github.ref == 'refs/heads/main'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{matrix.name}}--functional-test-xdc
|
|
path: /tmp/traces.json
|
|
|
|
functional-test-ndc:
|
|
if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
|
|
name: Functional test ndc
|
|
needs: [pre-build, set-up-single-test]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
name:
|
|
- cass_es
|
|
- cass_es8
|
|
- mysql8
|
|
- postgres12
|
|
- postgres12_pgx
|
|
include:
|
|
- name: cass_es
|
|
persistence_type: nosql
|
|
persistence_driver: elasticsearch
|
|
containers: [cassandra, elasticsearch]
|
|
es_version: v7
|
|
# Cassandra tests need a larger instance
|
|
runs_on_override: ubuntu-latest-8-cores
|
|
- name: cass_es8
|
|
persistence_type: nosql
|
|
persistence_driver: elasticsearch
|
|
containers: [cassandra, elasticsearch8]
|
|
es_version: v8
|
|
# Cassandra tests need a larger instance
|
|
runs_on_override: ubuntu-latest-8-cores
|
|
- name: mysql8
|
|
persistence_type: sql
|
|
persistence_driver: mysql8
|
|
containers: [mysql]
|
|
- name: postgres12
|
|
persistence_type: sql
|
|
persistence_driver: postgres12
|
|
containers: [postgresql]
|
|
- name: postgres12_pgx
|
|
persistence_type: sql
|
|
persistence_driver: postgres12_pgx
|
|
containers: [postgresql]
|
|
runs-on: ${{ matrix.runs_on_override || 'ubuntu-latest' }}
|
|
env:
|
|
PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
|
|
PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
|
|
ES_VERSION: ${{ matrix.es_version }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
ref: ${{ env.COMMIT }}
|
|
|
|
- name: Start containerized dependencies
|
|
if: ${{ toJson(matrix.containers) != '[]' }}
|
|
uses: hoverkraft-tech/compose-action@v2.0.1
|
|
with:
|
|
compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
|
|
services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}"
|
|
down-flags: -v
|
|
|
|
- uses: actions/setup-go@v5
|
|
with:
|
|
go-version-file: "go.mod"
|
|
cache: false # do our own caching
|
|
|
|
- name: Restore dependencies
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/go/pkg/mod
|
|
key: go-${{ runner.os }}${{ runner.arch }}-deps-${{ hashFiles('go.sum') }}
|
|
|
|
- name: Restore build outputs
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: ~/.cache/go-build
|
|
key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
|
|
|
|
- name: Get job ID from workflow run
|
|
id: get_job_id
|
|
run: |
|
|
set -eu
|
|
test_job_name="Functional test ndc (${{matrix.name}})"
|
|
job_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100"
|
|
# curl or jq might silently fail, but we are okay with that.
|
|
# We don't want to fail the job if we can't get the job id.
|
|
job_id=$(curl -L \
|
|
-H "Accept: application/vnd.github+json" \
|
|
-H "Authorization: Bearer ${{secrets.GITHUB_TOKEN}}" \
|
|
-H "X-GitHub-Api-Version: 2022-11-28" \
|
|
"$job_url" \
|
|
| jq "first(.jobs[] | select(.name == \"$test_job_name\") | .id)")
|
|
echo "Job id is $job_id"
|
|
echo "JOB_ID=${job_id:-unknown}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Run functional test ndc
|
|
timeout-minutes: 15
|
|
run: make OTEL=true functional-test-ndc-coverage
|
|
|
|
- name: Upload code coverage to Codecov
|
|
uses: codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: functional-test-ndc
|
|
|
|
- name: Upload test results to Codecov
|
|
if: ${{ !cancelled() }}
|
|
uses: codecov/test-results-action@v1
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
directory: ./.testoutput
|
|
flags: functional-test-ndc
|
|
|
|
- name: Upload test results to GitHub
|
|
# Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
|
|
uses: actions/upload-artifact@v4.4.3
|
|
if: ${{ !cancelled() }}
|
|
with:
|
|
name: junit-xml--${{github.run_id}}--${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{matrix.name}}--functional-test-ndc
|
|
path: .testoutput
|
|
include-hidden-files: true
|
|
retention-days: 28
|
|
|
|
# Ensure this doesn't contribute to the junit output.
|
|
- name: Flaky Functional NDC Test Detection
|
|
if: ${{ env.MODIFIED_TEST_SUITES != '' }}
|
|
timeout-minutes: 30
|
|
run: |
|
|
echo "Detecting flaky functional ndc tests: ${{ needs.set-up-single-test.outputs.modified_functional_ndc_test_suites }}"
|
|
make functional-test-ndc-coverage
|
|
env:
|
|
FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
|
|
TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_ndc_test_suites }} -count=5"
|
|
|
|
# Upload OpenTelemetry traces.
|
|
- name: Upload OpenTelemetry traces
|
|
if: always() && github.ref == 'refs/heads/main'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{matrix.name}}--functional-test-ndc
|
|
path: /tmp/traces.json
|
|
|
|
test-status:
|
|
if: always()
|
|
name: Test Status
|
|
needs:
|
|
- misc-checks
|
|
- unit-test
|
|
- integration-test
|
|
- functional-test
|
|
- functional-test-xdc
|
|
- functional-test-ndc
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
RESULTS: ${{ toJSON(needs.*.result) }}
|
|
steps:
|
|
- name: Check results
|
|
run: |
|
|
if [[ "${{ inputs.run_single_functional_test }}" != "true" ]]; then
|
|
# if in all-tests mode, all statuses must be success
|
|
if [[ -n $(echo "$RESULTS" | jq '.[] | select (. != "success")') ]]; then
|
|
exit 1
|
|
fi
|
|
else
|
|
# if in single-test mode, skipped status is ok, failure is failure
|
|
if [[ -n $(echo "$RESULTS" | jq '.[] | select (. == "failure")') ]]; then
|
|
exit 1
|
|
fi
|
|
fi |