Track flaky tests (redux) (#6049)

## What changed?
<!-- Describe what has changed in this PR -->

Uploading test results to Buildkite for analytical purposes.

⚠️ This will not work for any PR from a fork (Temporal developer or
not), due to GitHub's security model. It has to be a branch on this repo
to run successfully (includes `main`, ofc).

## Why?
<!-- Tell your future self why have you made these changes -->

To be able to analyze test flakiness.

## How did you test it?
<!-- How have you verified this change? Tested locally? Added a unit
test? Checked in staging env? -->

Ran this PR and saw test run in Buildkite.

<img width="1261" alt="Screenshot 2024-05-31 at 5 11 07 PM"
src="https://github.com/temporalio/temporal/assets/159852/f04f7392-12e2-4325-a161-ea038e9fe281">

## Potential risks
<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->

## Documentation
<!-- Have you made sure this change doesn't falsify anything currently
stated in `docs/`? If significant
new behavior is added, have you described that in `docs/`? -->

## Is hotfix candidate?
<!-- Is this PR a hotfix candidate or does it require a notification to
be sent to the broader community? (Yes/No) -->

.github/workflows/run-tests.yml

@@ -22,6 +22,7 @@ env:
   COMMIT: ${{ github.event.inputs.commit || github.sha }}
   DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
   TEMPORAL_VERSION_CHECK_DISABLED: 1
+  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
 
 jobs:
   misc-checks:
@@ -89,7 +90,7 @@ jobs:
 
       - name: Run unit test
         timeout-minutes: 15
-        run: make unit-test-coverage
+        run: make unit-test-coverage upload-test-results
 
   integration-test:
     name: Integration test
@@ -120,7 +121,7 @@ jobs:
 
       - name: Run integration test
         timeout-minutes: 15
-        run: make integration-test-coverage
+        run: make integration-test-coverage upload-test-results
 
       - name: Tear down docker compose
         if: ${{ always() }}
@@ -189,7 +190,7 @@ jobs:
 
       - name: Run functional test
         timeout-minutes: 25
-        run: make functional-test-coverage
+        run: make functional-test-coverage upload-test-results
 
       - name: Tear down docker compose
         if: ${{ always() }}
@@ -252,7 +253,7 @@ jobs:
 
       - name: Run functional test xdc
         timeout-minutes: 15
-        run: make functional-test-xdc-coverage
+        run: make functional-test-xdc-coverage upload-test-results
 
       - name: Tear down docker compose
         if: ${{ always() }}

Makefile

@@ -446,6 +446,9 @@ coverage-report: $(SUMMARY_COVER_PROFILE)
 	@printf $(COLOR) "Generate HTML report from $(SUMMARY_COVER_PROFILE) to $(SUMMARY_COVER_PROFILE).html..."
 	@go tool cover -html=$(SUMMARY_COVER_PROFILE) -o $(SUMMARY_COVER_PROFILE).html
 
+upload-test-results:
+	@(cd $(TEST_OUTPUT_ROOT) && sh $(ROOT)/develop/upload-test-results.sh)
+
 ##### Schema #####
 install-schema-cass-es: temporal-cassandra-tool install-schema-es
 	@printf $(COLOR) "Install Cassandra schema..."

develop/upload-test-results.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+if [ -z "$BUILDKITE_ANALYTICS_TOKEN" ]; then
+  echo "BUILDKITE_ANALYTICS_TOKEN is not set. Skipping."
+  exit 0  # we don't want the script to fail here
+fi
+
+echo "uploading test results from $(pwd)"
+
+for file in *.junit.xml; do
+    echo "uploading ${file}"
+
+    curl -i -X POST \
+      -H "Authorization: Token token=${BUILDKITE_ANALYTICS_TOKEN}" \
+      -F "data=@${file}" \
+      -F "format=junit" \
+      -F "run_env[CI]=github_actions" \
+      -F "run_env[key]=${GITHUB_ACTION}-${GITHUB_RUN_NUMBER}-${GITHUB_RUN_ATTEMPT}" \
+      -F "run_env[number]=${GITHUB_RUN_NUMBER}" \
+      -F "run_env[branch]=${GITHUB_REF}" \
+      -F "run_env[commit_sha]=${GITHUB_SHA}" \
+      -F "run_env[url]=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
+      https://analytics-api.buildkite.com/v1/uploads
+done