diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 014d7af7a1..8b8503ddfe 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -529,12 +529,16 @@ jobs: - name: Run functional test if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }} timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile - run: make functional-test-coverage + run: ./develop/github/monitor_test.sh make functional-test-coverage env: TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }} TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput TEMPORAL_OTEL_DEBUG: true + - name: Print memory snapshot + if: always() + run: if [ -f /tmp/memory_snapshot.txt ]; then cat /tmp/memory_snapshot.txt; fi + - name: Generate crash report if: failure() run: @@ -609,7 +613,14 @@ jobs: strategy: fail-fast: false matrix: - name: [cass_es, cass_es8, cass_os2, cass_os3, mysql8, postgres12, postgres12_pgx] + name: + - cass_es + - cass_es8 + - cass_os2 + - cass_os3 + - mysql8 + - postgres12 + - postgres12_pgx include: - name: cass_es persistence_type: nosql @@ -693,11 +704,15 @@ jobs: - name: Run functional test xdc timeout-minutes: 35 # update this to TEST_TIMEOUT+5 if you update the Makefile - run: make functional-test-xdc-coverage + run: ./develop/github/monitor_test.sh make functional-test-xdc-coverage env: TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput TEMPORAL_OTEL_DEBUG: true + - name: Print memory snapshot + if: always() + run: if [ -f /tmp/memory_snapshot.txt ]; then cat /tmp/memory_snapshot.txt; fi + - name: Generate crash report if: failure() run: @@ -859,11 +874,15 @@ jobs: - name: Run functional test ndc timeout-minutes: 15 - run: make functional-test-ndc-coverage + run: ./develop/github/monitor_test.sh make functional-test-ndc-coverage env: TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput TEMPORAL_OTEL_DEBUG: true + - name: Print memory snapshot + if: always() + run: if [ -f /tmp/memory_snapshot.txt ]; then cat /tmp/memory_snapshot.txt; fi + - name: Generate crash report if: failure() run: @@ -963,7 +982,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version-file: "go.mod" cache: true - name: Send Slack notification diff --git a/develop/github/memory_monitor.sh b/develop/github/memory_monitor.sh new file mode 100755 index 0000000000..0673d7953c --- /dev/null +++ b/develop/github/memory_monitor.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Memory Monitor +# +# Takes periodic memory snapshots. Captures system memory stats, top processes, +# and for Go processes, heap profiles via pprof. +# +# Usage: +# ./memory_monitor.sh +# +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +SNAPSHOT_FILE="$1" +HISTORY_FILE="/tmp/memory_history.txt" + +# Clear history on start +: > "$HISTORY_FILE" + +write_snapshot() { + local memtotal_kb memavail_kb memused_kb memused_mb pct + memtotal_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo)" + memavail_kb="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)" + memused_kb=$(( memtotal_kb - memavail_kb )) + memused_mb=$(( memused_kb / 1024 )) + pct=$(( memused_kb * 100 / memtotal_kb )) + + local goroutines="?" + if curl -s --max-time 5 'http://localhost:7000/debug/pprof/goroutine?debug=1' -o /tmp/goroutine.out 2>/dev/null; then + goroutines="$(head -1 /tmp/goroutine.out | grep -o '[0-9]*' || echo '?')" + fi + + # Get processes with >=1% memory, format as "name (MB)" + local top_procs + top_procs="$(ps -eo %mem,rss,comm --sort=-%mem | awk 'NR>1 && $1>=1.0 {printf "%s (%dMB), ", $3, $2/1024}' | sed 's/, $//')" + + local timestamp + timestamp="$(date '+%H:%M:%S')" + + # Append to history + echo "$timestamp $pct $memused_mb $goroutines $top_procs" >> "$HISTORY_FILE" + + { + echo "Memory snapshot at $(date '+%Y-%m-%d %H:%M:%S')" + echo "" + echo "Time Used(%) Used(MB) Goroutines Top Processes" + echo "---------------------------------------------------------------" + while read -r t p mb g procs; do + printf "%-10s %3s%% %5s %7s %s\n" "$t" "$p" "$mb" "$g" "$procs" + done < "$HISTORY_FILE" + echo "" + echo "--- Top Processes ---" + ps -eo pid,%mem,rss:10,comm --sort=-%mem | head -20 + echo "" + echo "--- Memory Summary ---" + free -m + echo "" + echo "--- Go Heap Profile ---" + if curl -s --max-time 10 "http://localhost:7000/debug/pprof/heap" -o /tmp/heap.out 2>/dev/null; then + go tool pprof -top -inuse_space /tmp/heap.out 2>/dev/null | head -60 || true + echo "" + go tool pprof -top -alloc_space /tmp/heap.out 2>/dev/null | head -60 || true + else + echo "(pprof endpoint not available)" + fi + } > "$SNAPSHOT_FILE" +} + +# Take snapshots every 30s until killed +while true; do + write_snapshot + sleep 30 +done diff --git a/develop/github/monitor_test.sh b/develop/github/monitor_test.sh new file mode 100755 index 0000000000..6c0f773b89 --- /dev/null +++ b/develop/github/monitor_test.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Monitor Test +# +# Runs the given command with background monitoring. +# +# Usage: +# ./monitor_test.sh [args...] +# +# Example: +# ./monitor_test.sh make functional-test-coverage +# +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [args...]" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Start monitor +"$SCRIPT_DIR/memory_monitor.sh" /tmp/memory_snapshot.txt & +MONITOR_PID=$! +trap 'kill "$MONITOR_PID" 2>/dev/null' EXIT + +# Run command +"$@"