Github/temporal
1
0
Fork 0
mirror of https://github.com/temporalio/temporal.git synced 2026-08-30 18:41:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

Action inputs to dispatch n-runs of a single test in CI (#6297)

Browse Source 
## What changed?
Add workflow dispatch options to the functional tests Github Action to
allow us to run n-iterations of a single functional test with a
configurable timeout.

There is also an option to run n-iterations of a single unit test,
although it may be faster to run that locally.

WARNING: For functional tests, this will definitely be oomkilled for
n>=100, likely for n=>50 too. I suggest to start with n=20 to see how
the memory goes and then increase from there. Different DBs may use
different amounts of RAM also.

## Why?
To aid in the diagnosis and treatment of flaky tests.

## How did you test it?
Tested in github actions.
Here is the action run normally, with no new input parameters:
https://github.com/temporalio/temporal/actions/runs/11261156403
Here is the action run on one test multiple times:
https://github.com/temporalio/temporal/actions/runs/11261236304

While we still have buildkite, the uploaded test results will be
uploaded. You can find them by going to
https://buildkite.com/organizations/temporal/analytics/suites/temporal-public/runs?branch=all+branches
and looking for a recent run with `"job: functional-test"` and the
commit hash you used.
Here is the buildkite output of the run above:
https://buildkite.com/organizations/temporal/analytics/suites/temporal-public/runs/39901afd-35f1-8171-9257-e0bada374824

## Potential risks
Our functional test pipeline could be broken by this PR, but we would
notice that pretty immediately

## Documentation
How to run it yourself
1. Go to
https://github.com/temporalio/temporal/actions/workflows/run-tests.yml
2. Click "run workflow" on the upper right hand side
5. Set Commit SHA to the latest commit on the branch
6. Select your desired options
7. Click the green "Run workflow" button

## Is hotfix candidate?
<!-- Is this PR a hotfix candidate or does it require a notification to
be sent to the broader community? (Yes/No) -->
This commit is contained in:
Carly de Frondeville
2024-10-09 15:17:46 -04:00
committed by GitHub
parent a5e8be9005
commit 65a58d987c
3 changed files with 137 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
.github/workflows/run-tests.yml vendored
View File

@@ -13,6 +13,40 @@ on:
commit:
description: "Commit SHA"
required: true
run_single_functional_test:
description: "Run a single functional test"
type: boolean
default: false
run_single_unit_test:
description: "Run a single unit test (INSTEAD of functional test)"
type: boolean
default: false
unit_test_directory:
description: "[Unit Test Only] Directory to run unit tests in"
type: string
default: "./temporal"
n_runs:
description: "Number of times to repeat the single test per database type"
type: number
default: 1
test_name:
description: "Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
type: string
timeout_minutes:
description: "Test timeout in minutes"
type: number
default: 120
test_runner:
description: "Which runner to use. Choose higher RAM if your n_runs is high."
type: choice
default: "16GB RAM (ubuntu-20.04)"
options:
- "16GB RAM (ubuntu-20.04)"
- "64GB RAM (ubuntu-22.04)"
test_dbs:
description: 'List of DBs to test on (i.e. ["sqlite", "cassandra", "mysql8", "postgres12"])'
type: string
default: '["sqlite"]'
concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
group: run-tests-${{ github.head_ref || github.run_id }}
@@ -29,6 +63,43 @@ env:
BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
jobs:
set-up-single-test:
name: Set up single test
runs-on: ubuntu-20.04
outputs:
shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
total_shards: ${{ steps.generate_output.outputs.shards }}
github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
runs_on: ${{ steps.generate_output.outputs.runs_on }}
dbs: ${{ inputs.test_dbs }}
steps:
- id: generate_output
run: |
shards=3
timeout=30
runs_on='["ubuntu-20.04"]'
if [[ "${{ inputs.run_single_functional_test }}" == "true" || "${{ inputs.run_single_unit_test }}" == "true" ]]; then
shards=1
timeout=${{ inputs.timeout_minutes }}
single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
runs_on='[ "ubuntu-latest-16-cores" ]'
fi
fi
{
echo "shard_indices=[ $(seq -s, 0 $((shards-1))) ]"
echo "shards=$shards"
echo "github_timeout=$((timeout+5))"
echo "test_timeout=${timeout}m"
echo "single_test_args=$single_test_args"
echo "runs_on=$runs_on"
} >> "$GITHUB_OUTPUT"
- id: cat_output
run: |
cat "$GITHUB_OUTPUT"
misc-checks:
name: Misc checks
strategy:
@@ -38,6 +109,7 @@ jobs:
runs-on: ${{ matrix.runs-on }}
steps:
- uses: actions/checkout@v4
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
with:
token: ${{ secrets.GITHUB_TOKEN }}
ref: ${{ env.COMMIT }}
@@ -45,28 +117,41 @@ jobs:
fetch-depth: 100
- uses: actions/setup-go@v5
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
with:
go-version-file: 'go.mod'
check-latest: true
- uses: arduino/setup-protoc@v3
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
- run: GOOS=windows GOARCH=amd64 make clean-bins bins
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
- run: GOOS=darwin GOARCH=arm64 make clean-bins bins
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
- run: make clean-bins ci-build-misc
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
- run: make build-tests
if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
unit-test:
if: ${{ inputs.run_single_functional_test != true }}
name: Unit test
needs: misc-checks
needs: [misc-checks, set-up-single-test]
strategy:
fail-fast: false
matrix:
runs-on: [ubuntu-20.04]
runs-on: ${{ matrix.runs-on }}
env:
BUILDKITE_MESSAGE: "{\"job\": \"unit-test\"}"
SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
UNIT_TEST_DIR: ${{ inputs.unit_test_directory }}
TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
RUN_SINGLE_UNIT_TEST: ${{ inputs.run_single_unit_test }}
steps:
- uses: actions/checkout@v4
with:
@@ -78,15 +163,22 @@ jobs:
go-version-file: 'go.mod'
check-latest: true
- name: Run unit test
- name: Run unit tests
if: ${{ !cancelled() && !inputs.run_single_unit_test }}
timeout-minutes: 15
run: make unit-test-coverage
- name: Run single unit test
if: ${{ !cancelled() && inputs.run_single_unit_test }}
timeout-minutes: 15
run: UNIT_TEST_DIRS=$UNIT_TEST_DIR make unit-test-coverage
- name: Upload test results
if: ${{ !cancelled() }}
run: make upload-test-results
integration-test:
if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
name: Integration test
needs: misc-checks
strategy:
@@ -94,6 +186,8 @@ jobs:
matrix:
runs-on: [ubuntu-20.04]
runs-on: ${{ matrix.runs-on }}
env:
BUILDKITE_MESSAGE: "{\"job\": \"integration-test\"}"
steps:
- uses: actions/checkout@v4
with:
@@ -123,13 +217,20 @@ jobs:
if: ${{ !cancelled() }}
run: make upload-test-results
- name: Tear down docker compose
if: ${{ always() }}
run: |
docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
functional-test:
if: ${{ inputs.run_single_unit_test != true }}
name: Functional test
needs: misc-checks
needs: [misc-checks, set-up-single-test]
strategy:
fail-fast: false
matrix:
runs-on: [ubuntu-20.04]
runs-on: ${{ fromJson(needs.set-up-single-test.outputs.runs_on) }}
shard_index: ${{ fromJson(needs.set-up-single-test.outputs.shard_indices) }}
name:
- cass_es
- cass_es8
@@ -137,7 +238,6 @@ jobs:
- mysql8
- postgres12
- postgres12_pgx
shard_index: [0, 1, 2]
include:
- name: cass_es
persistence_type: nosql
@@ -167,23 +267,35 @@ jobs:
containers: [postgresql]
runs-on: ${{ matrix.runs-on }}
env:
TEST_TOTAL_SHARDS: 3
TEST_TOTAL_SHARDS: ${{ needs.set-up-single-test.outputs.total_shards }}
TEST_SHARD_INDEX: ${{ matrix.shard_index }}
PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
steps:
- name: Print functional test
run: echo "${{ needs.set-up-single-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS"
- uses: ScribeMD/docker-cache@0.3.7
if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
with:
key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
- uses: actions/checkout@v4
if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
with:
token: ${{ secrets.GITHUB_TOKEN }}
ref: ${{ env.COMMIT }}
- uses: actions/setup-go@v5
if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
with:
go-version-file: 'go.mod'
check-latest: true
- name: Start containerized dependencies
if: ${{ toJson(matrix.containers) != '[]' }}
if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
uses: hoverkraft-tech/compose-action@v2.0.1
with:
compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
@@ -191,14 +303,16 @@ jobs:
down-flags: -v
- name: Run functional test
timeout-minutes: 30 # make sure this is larger than the test timeout in the Makefile
if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
run: make functional-test-coverage
- name: Upload test results
if: ${{ !cancelled() }}
if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
run: make upload-test-results
functional-test-xdc:
if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
name: Functional test xdc
needs: misc-checks
strategy:
@@ -231,6 +345,7 @@ jobs:
env:
PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
BUILDKITE_MESSAGE: "{\"job\": \"functional-test-xdc\", \"db\": \"${{ matrix.persistence_driver }}\"}"
steps:
- uses: actions/checkout@v4
with:
@@ -259,13 +374,14 @@ jobs:
run: make upload-test-results
functional-test-ndc:
if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
name: Functional test ndc
needs: misc-checks
strategy:
fail-fast: false
matrix:
runs-on: [ubuntu-20.04]
name:
name:
- cass_es
- cass_es8
- mysql8
@@ -323,6 +439,7 @@ jobs:
run: make functional-test-ndc-coverage
test-status:
if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
name: Test Status
needs:
- unit-test
@@ -331,7 +448,6 @@ jobs:
- functional-test-xdc
- functional-test-ndc
runs-on: ubuntu-20.04
if: always()
env:
RESULTS: ${{ toJSON(needs.*.result) }}
steps:

14
Makefile
View File

@@ -64,7 +64,7 @@ define NEWLINE
endef
TEST_TIMEOUT := 25m
TEST_TIMEOUT ?= 25m
PROTO_ROOT := proto
PROTO_FILES = $(shell find ./$(PROTO_ROOT)/internal -name "*.proto")
@@ -88,7 +88,7 @@ FUNCTIONAL_TEST_NDC_ROOT := ./tests/ndc
DB_INTEGRATION_TEST_ROOT := ./common/persistence/tests
DB_TOOL_INTEGRATION_TEST_ROOT := ./tools/tests
INTEGRATION_TEST_DIRS := $(DB_INTEGRATION_TEST_ROOT) $(DB_TOOL_INTEGRATION_TEST_ROOT) ./temporaltest ./internal/temporalite
UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% ./internal/temporalite%,$(TEST_DIRS))
UNIT_TEST_DIRS ?= $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% ./internal/temporalite%,$(TEST_DIRS))
# github.com/urfave/cli/v2@v2.4.0 - needs to accept comma in values before unlocking https://github.com/urfave/cli/pull/1241.
PINNED_DEPENDENCIES := \
@@ -381,8 +381,8 @@ prepare-coverage-test: $(GOTESTSUM) $(TEST_OUTPUT_ROOT)
unit-test-coverage: prepare-coverage-test
@printf $(COLOR) "Run unit tests with coverage..."
@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
$(UNIT_TEST_DIRS) -shuffle on -timeout=$(TEST_TIMEOUT) -race $(TEST_TAG_FLAG) -coverprofile=$(NEW_COVER_PROFILE)
$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
$(UNIT_TEST_DIRS) -shuffle on -timeout=$(TEST_TIMEOUT) -race $(TEST_TAG_FLAG) $(SINGLE_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE)
integration-test-coverage: prepare-coverage-test
@printf $(COLOR) "Run integration tests with coverage..."
@@ -395,8 +395,10 @@ pre-build-functional-test-coverage: prepare-coverage-test
functional-test-coverage: prepare-coverage-test
@printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..."
@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(SINGLE_TEST_ARGS) $(TEST_TAG_FLAG) \
-persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) \
$(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
functional-test-xdc-coverage: prepare-coverage-test
@printf $(COLOR) "Run functional test for cross DC with coverage with $(PERSISTENCE_DRIVER) driver..."

3
develop/upload-test-results.sh
View File

@@ -21,5 +21,6 @@ for file in *.junit.xml; do
-F "run_env[branch]=${GITHUB_REF}" \
-F "run_env[commit_sha]=${GITHUB_SHA}" \
-F "run_env[url]=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
-F "run_env[message]=${BUILDKITE_MESSAGE}" \
https://analytics-api.buildkite.com/v1/uploads
done
done