Collect OTEL traces from CI tests (#7871)

## What changed? Collecting OTEL traces from functional tests. And packaging them up as an artifact. ## Why Help debug flaky tests. ## Testing You can see the trace artifacts on [this run](https://github.com/temporalio/temporal/actions/runs/15719131802?pr=7871).
2026-08-30 18:41:49 -07:00 · 2025-06-18 09:10:52 -07:00
parent 4c802e02ba
commit 80a957acec
2 changed files with 79 additions and 16 deletions
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -306,8 +306,7 @@ jobs:
          echo "Detecting flaky unit tests: ${{ needs.set-up-single-test.outputs.modified_unit_test_suites }}"
          make unit-test
        env:
-          # Not retrying failed tests intentionally here since we're trying to detect flakes.
-          FAILED_TEST_RETRIES: "0"
+          FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
          TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_unit_test_suites }} -count=10"
          TEST_TIMEOUT: 35

@@ -398,8 +397,7 @@ jobs:
          echo "Detecting flaky integration tests: ${{ needs.set-up-single-test.outputs.modified_integration_test_suites }}"
          make integration-test-coverage
        env:
-          # Not retrying failed tests intentionally here since we're trying to detect flakes.
-          FAILED_TEST_RETRIES: "0"
+          FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
          TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_integration_test_suites }} -count=5"

      - name: Tear down docker compose
@@ -478,7 +476,7 @@ jobs:
        uses: hoverkraft-tech/compose-action@v2.0.1
        with:
          compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
-          services: "${{ join(matrix.containers, '\n') }}"
+          services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}"
          down-flags: -v

      - uses: actions/setup-go@v5
@@ -519,7 +517,7 @@ jobs:
      - name: Run functional test
        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
        timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
-        run: make functional-test-coverage
+        run: make OTEL=true functional-test-coverage
        env:
          TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}

@@ -566,10 +564,17 @@ jobs:
          echo "Detecting flaky functional tests: ${{ needs.set-up-single-test.outputs.modified_integration_test_suites }}"
          make functional-test-coverage
        env:
-          # Not retrying failed tests intentionally here since we're trying to detect flakes.
-          FAILED_TEST_RETRIES: "0"
+          FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
          TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_test_suites }} -count=5"

+      # Upload OpenTelemetry traces.
+      - name: Upload OpenTelemetry traces
+        if: always() && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@v4
+        with:
+          name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{matrix.shard_index}}--${{github.run_attempt}}--${{matrix.name}}--functional-test
+          path: /tmp/traces.json
+
  functional-test-xdc:
    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
    name: Functional test xdc
@@ -624,7 +629,7 @@ jobs:
        uses: hoverkraft-tech/compose-action@v2.0.1
        with:
          compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
-          services: "${{ join(matrix.containers, '\n') }}"
+          services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}"
          down-flags: -v

      - uses: actions/setup-go@v5
@@ -663,7 +668,7 @@ jobs:

      - name: Run functional test xdc
        timeout-minutes: 35   # update this to TEST_TIMEOUT+5 if you update the Makefile
-        run: make functional-test-xdc-coverage
+        run: make OTEL=true functional-test-xdc-coverage

      - name: Generate test summary
        uses: mikepenz/action-junit-report@v5.0.0-rc01
@@ -708,10 +713,17 @@ jobs:
          echo "Detecting flaky functional xdc tests: ${{ needs.set-up-single-test.outputs.modified_functional_xdc_test_suites }}"
          make functional-test-xdc-coverage
        env:
-          # Not retrying failed tests intentionally here since we're trying to detect flakes.
-          FAILED_TEST_RETRIES: "0"
+          FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
          TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_xdc_test_suites }} -count=5"

+      # Upload OpenTelemetry traces.
+      - name: Upload OpenTelemetry traces
+        if: always() && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@v4
+        with:
+          name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{matrix.name}}--functional-test-xdc
+          path: /tmp/traces.json
+
  functional-test-ndc:
    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
    name: Functional test ndc
@@ -768,7 +780,7 @@ jobs:
        uses: hoverkraft-tech/compose-action@v2.0.1
        with:
          compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
-          services: "${{ join(matrix.containers, '\n') }}"
+          services: "${{ format('{0}\notel-collector', join(matrix.containers, '\n')) }}"
          down-flags: -v

      - uses: actions/setup-go@v5
@@ -807,7 +819,7 @@ jobs:

      - name: Run functional test ndc
        timeout-minutes: 15
-        run: make functional-test-ndc-coverage
+        run: make OTEL=true functional-test-ndc-coverage

      - name: Upload code coverage to Codecov
        uses: codecov/codecov-action@v5
@@ -842,10 +854,17 @@ jobs:
          echo "Detecting flaky functional ndc tests: ${{ needs.set-up-single-test.outputs.modified_functional_ndc_test_suites }}"
          make functional-test-ndc-coverage
        env:
-          # Not retrying failed tests intentionally here since we're trying to detect flakes.
-          FAILED_TEST_RETRIES: "0"
+          FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes
          TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_ndc_test_suites }} -count=5"

+      # Upload OpenTelemetry traces.
+      - name: Upload OpenTelemetry traces
+        if: always() && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@v4
+        with:
+          name: traces-json--${{github.run_id}}-${{steps.get_job_id.outputs.JOB_ID}}--${{github.run_attempt}}--${{matrix.name}}--functional-test-ndc
+          path: /tmp/traces.json
+
  test-status:
    if: always()
    name: Test Status
--- a/develop/github/docker-compose.yml
+++ b/develop/github/docker-compose.yml
@@ -51,3 +51,47 @@ services:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - ES_JAVA_OPTS=-Xms1g -Xmx1g
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.127.0
+    ports:
+      - "4317:4317"
+    volumes:
+      - /tmp:/out:rw
+    environment:
+      OTEL_CONFIG: |
+        receivers:
+          otlp:
+            protocols:
+              grpc:
+                endpoint: 0.0.0.0:4317
+        processors:
+          batch:
+            timeout: 1s          
+            send_batch_size: 100
+            send_batch_max_size: 1000
+          # Remove superfluous attributes to save space
+          transform:
+            trace_statements:
+              - context: resource
+                statements:
+                  - delete_matching_keys(attributes, "host\\..*")
+                  - delete_matching_keys(attributes, "os\\..*")
+                  - delete_matching_keys(attributes, "process\\..*")
+                  - delete_matching_keys(attributes, "service.version")
+              - context: span
+                statements:
+                  - delete_matching_keys(attributes, "net\\..*")
+                  - delete_matching_keys(attributes, "rpc.system")
+                  - delete_matching_keys(attributes, "rpc.request.headers\\..*")
+        exporters:
+          file:
+            path: /out/traces.json
+            flush_interval: 1s
+        service:
+          pipelines:
+            traces:
+              receivers: [otlp]
+              processors: [transform, batch]
+              exporters: [file]
+    command: [ "--config", "env:OTEL_CONFIG" ]