diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 9b3b631bd7..6e0312f4d9 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -306,8 +306,7 @@ jobs: echo "Detecting flaky unit tests: ${{ needs.set-up-single-test.outputs.modified_unit_test_suites }}" make unit-test env: - # Not retrying failed tests intentionally here since we're trying to detect flakes. - FAILED_TEST_RETRIES: "0" + FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_unit_test_suites }} -count=10" TEST_TIMEOUT: 35 @@ -398,8 +397,7 @@ jobs: echo "Detecting flaky integration tests: ${{ needs.set-up-single-test.outputs.modified_integration_test_suites }}" make integration-test-coverage env: - # Not retrying failed tests intentionally here since we're trying to detect flakes. - FAILED_TEST_RETRIES: "0" + FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_integration_test_suites }} -count=5" - name: Tear down docker compose @@ -478,7 +476,7 @@ jobs: uses: hoverkraft-tech/compose-action@v2.0.1 with: compose-file: ${{ env.DOCKER_COMPOSE_FILE }} - services: "${{ join(matrix.containers, '\n') }}" + services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}" down-flags: -v - uses: actions/setup-go@v5 @@ -519,7 +517,7 @@ jobs: - name: Run functional test if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }} timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile - run: make functional-test-coverage + run: make OTEL=true functional-test-coverage env: TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }} @@ -566,10 +564,17 @@ jobs: echo "Detecting flaky functional tests: ${{ needs.set-up-single-test.outputs.modified_integration_test_suites }}" make functional-test-coverage env: - # Not retrying failed tests intentionally here since we're trying to detect flakes. - FAILED_TEST_RETRIES: "0" + FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_test_suites }} -count=5" + # Upload OpenTelemetry traces. + - name: Upload OpenTelemetry traces + if: always() && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v4 + with: + name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{matrix.shard_index}}--${{github.run_attempt}}--${{matrix.name}}--functional-test + path: /tmp/traces.json + functional-test-xdc: if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }} name: Functional test xdc @@ -624,7 +629,7 @@ jobs: uses: hoverkraft-tech/compose-action@v2.0.1 with: compose-file: ${{ env.DOCKER_COMPOSE_FILE }} - services: "${{ join(matrix.containers, '\n') }}" + services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}" down-flags: -v - uses: actions/setup-go@v5 @@ -663,7 +668,7 @@ jobs: - name: Run functional test xdc timeout-minutes: 35 # update this to TEST_TIMEOUT+5 if you update the Makefile - run: make functional-test-xdc-coverage + run: make OTEL=true functional-test-xdc-coverage - name: Generate test summary uses: mikepenz/action-junit-report@v5.0.0-rc01 @@ -708,10 +713,17 @@ jobs: echo "Detecting flaky functional xdc tests: ${{ needs.set-up-single-test.outputs.modified_functional_xdc_test_suites }}" make functional-test-xdc-coverage env: - # Not retrying failed tests intentionally here since we're trying to detect flakes. - FAILED_TEST_RETRIES: "0" + FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_xdc_test_suites }} -count=5" + # Upload OpenTelemetry traces. + - name: Upload OpenTelemetry traces + if: always() && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v4 + with: + name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{matrix.name}}--functional-test-xdc + path: /tmp/traces.json + functional-test-ndc: if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }} name: Functional test ndc @@ -768,7 +780,7 @@ jobs: uses: hoverkraft-tech/compose-action@v2.0.1 with: compose-file: ${{ env.DOCKER_COMPOSE_FILE }} - services: "${{ join(matrix.containers, '\n') }}" + services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}" down-flags: -v - uses: actions/setup-go@v5 @@ -807,7 +819,7 @@ jobs: - name: Run functional test ndc timeout-minutes: 15 - run: make functional-test-ndc-coverage + run: make OTEL=true functional-test-ndc-coverage - name: Upload code coverage to Codecov uses: codecov/codecov-action@v5 @@ -842,10 +854,17 @@ jobs: echo "Detecting flaky functional ndc tests: ${{ needs.set-up-single-test.outputs.modified_functional_ndc_test_suites }}" make functional-test-ndc-coverage env: - # Not retrying failed tests intentionally here since we're trying to detect flakes. - FAILED_TEST_RETRIES: "0" + FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_ndc_test_suites }} -count=5" + # Upload OpenTelemetry traces. + - name: Upload OpenTelemetry traces + if: always() && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v4 + with: + name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{matrix.name}}--functional-test-ndc + path: /tmp/traces.json + test-status: if: always() name: Test Status diff --git a/develop/github/docker-compose.yml b/develop/github/docker-compose.yml index f35c169567..4e9bef2953 100644 --- a/develop/github/docker-compose.yml +++ b/develop/github/docker-compose.yml @@ -51,3 +51,47 @@ services: - discovery.type=single-node - xpack.security.enabled=false - ES_JAVA_OPTS=-Xms1g -Xmx1g + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.127.0 + ports: + - "4317:4317" + volumes: + - /tmp:/out:rw + environment: + OTEL_CONFIG: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + processors: + batch: + timeout: 1s + send_batch_size: 100 + send_batch_max_size: 1000 + # Remove superfluous attributes to save space + transform: + trace_statements: + - context: resource + statements: + - delete_matching_keys(attributes, "host\\..*") + - delete_matching_keys(attributes, "os\\..*") + - delete_matching_keys(attributes, "process\\..*") + - delete_matching_keys(attributes, "service.version") + - context: span + statements: + - delete_matching_keys(attributes, "net\\..*") + - delete_matching_keys(attributes, "rpc.system") + - delete_matching_keys(attributes, "rpc.request.headers\\..*") + exporters: + file: + path: /out/traces.json + flush_interval: 1s + service: + pipelines: + traces: + receivers: [otlp] + processors: [transform, batch] + exporters: [file] + command: [ "--config", "env:OTEL_CONFIG" ]