mirror of
https://github.com/temporalio/temporal.git
synced 2026-08-30 18:41:49 -07:00
Improve memory diagnostics for test jobs (#10742)
## What changed? - Poll CI memory every 1s with cheap process snapshots. - Capture pprof and process memory diagnostics when memory crosses threshold, then every 30s while above it. - Upload memory diagnostics artifacts from post-test reporting. ## Why? Make OOM failures easier to diagnose before the runner terminates the test process. Example: https://github.com/temporalio/temporal/actions/runs/29111099672/job/86424089913?pr=10742#step:7:77
This commit is contained in:
Stephan Behnke
@@ -2,147 +2,247 @@
|
||||
#
|
||||
# Memory Monitor
|
||||
#
|
||||
# Takes periodic memory snapshots. Captures system memory stats, top processes,
|
||||
# and for Go processes, heap profiles via pprof.
|
||||
# 1. Snapshot status:
|
||||
# Samples memory every SNAPSHOT_INTERVAL_SECONDS and writes every sample
|
||||
# to SNAPSHOT_HISTORY_FILE. Logs status every SNAPSHOT_PRINT_INTERVAL_SECONDS
|
||||
# and writes the highest-memory snapshot report to SNAPSHOT_FILE.
|
||||
#
|
||||
# 2. Profile capture:
|
||||
# When usage crosses HEAP_PROFILE_CAPTURE_THRESHOLD, captures a heap
|
||||
# profile in HEAP_PROFILES_DIR before running analysis.
|
||||
#
|
||||
# 3. OOM prevention:
|
||||
# When usage crosses OOM_TERMINATION_THRESHOLD, reuses any previously
|
||||
# captured profile, writes the latest snapshot and a synthetic JUnit
|
||||
# report, then terminates the monitored test process group so post-test
|
||||
# artifact upload can still run. If no profile has been captured yet, it
|
||||
# captures one before terminating.
|
||||
#
|
||||
# Usage:
|
||||
# ./memory_monitor.sh <snapshot-file>
|
||||
# ./memory_monitor.sh
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
echo "Usage: $0 <snapshot-file>" >&2
|
||||
if [[ $# -ne 0 ]]; then
|
||||
echo "Usage: $0" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SNAPSHOT_FILE="$1"
|
||||
HISTORY_FILE="/tmp/memory_history.txt"
|
||||
HIGH_MEMORY_THRESHOLD=95
|
||||
PPROF_HOST="${PPROF_HOST:-localhost:7000}"
|
||||
HEAP_PRINTED=false
|
||||
HIGH_WATER_MARK=0
|
||||
# Snapshot config.
|
||||
readonly SNAPSHOT_DIR="${SNAPSHOT_DIR:-.testoutput/memory}"
|
||||
# Sample every second so short OOM ramps still leave history, but print less
|
||||
# often to keep CI logs readable.
|
||||
readonly SNAPSHOT_INTERVAL_SECONDS="${SNAPSHOT_INTERVAL_SECONDS:-1}"
|
||||
readonly SNAPSHOT_PRINT_INTERVAL_SECONDS="${SNAPSHOT_PRINT_INTERVAL_SECONDS:-30}"
|
||||
readonly SNAPSHOT_FILE="${SNAPSHOT_FILE:-$SNAPSHOT_DIR/memory-snapshot.txt}"
|
||||
readonly SNAPSHOT_HISTORY_FILE="${SNAPSHOT_HISTORY_FILE:-$SNAPSHOT_DIR/memory-history.txt}"
|
||||
|
||||
# Clear history on start
|
||||
: > "$HISTORY_FILE"
|
||||
# Heap profile config.
|
||||
# Capture before the termination threshold so the diagnostic profile is usually
|
||||
# available even if the runner kills the job before our termination path runs.
|
||||
readonly HEAP_PROFILE_CAPTURE_THRESHOLD="${HEAP_PROFILE_CAPTURE_THRESHOLD:-90}"
|
||||
readonly HEAP_PROFILE_REFRESH_INTERVAL_SECONDS="${HEAP_PROFILE_REFRESH_INTERVAL_SECONDS:-30}"
|
||||
readonly HEAP_PROFILES_DIR="${HEAP_PROFILES_DIR:-$SNAPSHOT_DIR/heap-profiles}"
|
||||
readonly PPROF_HOST="${PPROF_HOST:-localhost:7000}"
|
||||
|
||||
# OOM prevention config.
|
||||
# Terminate late enough to avoid masking near-finished tests, but before the
|
||||
# runner OOM killer skips post-test artifact upload.
|
||||
readonly OOM_TERMINATION_THRESHOLD="${OOM_TERMINATION_THRESHOLD:-99}"
|
||||
readonly OOM_JUNIT_FILE="${OOM_JUNIT_FILE:-.testoutput/junit.oom.xml}"
|
||||
|
||||
# State.
|
||||
MEMORY_HIGH_WATER_MARK=0
|
||||
LAST_SNAPSHOT_PRINT_TIME=0
|
||||
LAST_HEAP_PROFILE_CAPTURE_TIME=0
|
||||
HEAP_PROFILE_SECTION=""
|
||||
HAS_CAPTURED_HEAP_PROFILE=false
|
||||
OOM_TERMINATED=false
|
||||
|
||||
ensure_snapshot_dirs() {
|
||||
mkdir -p "$(dirname "$SNAPSHOT_FILE")" "$(dirname "$SNAPSHOT_HISTORY_FILE")"
|
||||
}
|
||||
|
||||
init_snapshot_files() {
|
||||
ensure_snapshot_dirs
|
||||
: > "$SNAPSHOT_HISTORY_FILE"
|
||||
}
|
||||
|
||||
# Fetch a pprof profile and save to file
|
||||
# Usage: fetch_pprof <profile_type> <output_file>
|
||||
# Usage: fetch_pprof <pprof_profile> <output_file>
|
||||
# Returns 0 on success, 1 on failure
|
||||
fetch_pprof() {
|
||||
local profile_type="$1"
|
||||
local pprof_profile="$1"
|
||||
local output_file="$2"
|
||||
curl -s --max-time 10 "http://${PPROF_HOST}/debug/pprof/${profile_type}" -o "$output_file" 2>/dev/null
|
||||
}
|
||||
|
||||
# Print pprof top analysis.
|
||||
# Usage: pprof_top <profile_file> <lines> [extra_flags...]
|
||||
pprof_top() {
|
||||
local profile_file="$1"
|
||||
local lines="$2"
|
||||
shift 2
|
||||
|
||||
go tool pprof -top "$@" "$profile_file" 2>/dev/null | head -"$lines" || true
|
||||
}
|
||||
|
||||
# Print goroutine profile analysis.
|
||||
print_goroutines() {
|
||||
local tmp_file
|
||||
tmp_file="$(mktemp)"
|
||||
trap 'rm -f "$tmp_file"' RETURN
|
||||
|
||||
if fetch_pprof "goroutine" "$tmp_file"; then
|
||||
echo "=== top functions by goroutine count ==="
|
||||
pprof_top "$tmp_file" 30
|
||||
fi
|
||||
}
|
||||
|
||||
# Extract goroutine count from pprof output ("... of N total").
|
||||
count_goroutines() {
|
||||
sed -n 's/.*of \([0-9]*\) total.*/\1/p' | head -1 || echo '?'
|
||||
curl -s --max-time 10 "http://${PPROF_HOST}/debug/pprof/${pprof_profile}" -o "$output_file" 2>/dev/null
|
||||
}
|
||||
|
||||
# Print heap profile analysis.
|
||||
print_heap() {
|
||||
local tmp_file
|
||||
tmp_file="$(mktemp)"
|
||||
trap 'rm -f "$tmp_file"' RETURN
|
||||
print_heap_profile_summary() {
|
||||
local heap_profile_file="$1"
|
||||
|
||||
echo "--- Go Heap Profile ---"
|
||||
if fetch_pprof "heap" "$tmp_file"; then
|
||||
if [[ -s "$heap_profile_file" ]]; then
|
||||
echo "=== inuse_space (what's currently held) ==="
|
||||
pprof_top "$tmp_file" 30 -inuse_space
|
||||
echo ""
|
||||
echo "=== alloc_space (total allocations) ==="
|
||||
pprof_top "$tmp_file" 30 -alloc_space
|
||||
echo ""
|
||||
echo "=== alloc_objects (total objects allocated) ==="
|
||||
pprof_top "$tmp_file" 30 -alloc_objects
|
||||
# Keep the artifact focused on retained memory; allocation totals are noisy
|
||||
# for this CI OOM investigation.
|
||||
go tool pprof -top -inuse_space "$heap_profile_file" 2>/dev/null | head -15 || true
|
||||
else
|
||||
echo "(pprof endpoint not available)"
|
||||
echo "(heap profile not available)"
|
||||
fi
|
||||
}
|
||||
|
||||
snapshot() {
|
||||
local memtotal_kb memavail_kb memused_kb memused_mb pct
|
||||
memtotal_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo)"
|
||||
memavail_kb="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)"
|
||||
memused_kb=$(( memtotal_kb - memavail_kb ))
|
||||
memused_mb=$(( memused_kb / 1024 ))
|
||||
pct=$(( memused_kb * 100 / memtotal_kb ))
|
||||
terminate_monitored_processes() {
|
||||
local memory_pct="$1"
|
||||
|
||||
# Collect pprof data once per tick.
|
||||
local goroutine_output goroutine_count pprof_output
|
||||
goroutine_output="$(print_goroutines)"
|
||||
goroutine_count="$(count_goroutines <<< "$goroutine_output")"
|
||||
pprof_output="$(print_heap)"
|
||||
if [[ -n "${MONITORED_PROCESS_GROUP:-}" ]]; then
|
||||
echo "Terminating monitored process group ${MONITORED_PROCESS_GROUP} at ${memory_pct}% memory to preserve diagnostics artifacts."
|
||||
kill -TERM "-$MONITORED_PROCESS_GROUP" 2>/dev/null || true
|
||||
return
|
||||
fi
|
||||
|
||||
# Get processes with >=1% memory, format as "name (MB)"
|
||||
local top_procs
|
||||
top_procs="$(ps -eo %mem,rss,comm --sort=-%mem | awk 'NR>1 && $1>=1.0 {printf "%s (%dMB), ", $3, $2/1024}' | sed 's/, $//')"
|
||||
echo "No monitored process group set at ${memory_pct}% memory; leaving processes running."
|
||||
}
|
||||
|
||||
local timestamp
|
||||
timestamp="$(date '+%H:%M:%S')"
|
||||
write_oom_junit() {
|
||||
local memory_pct="$1"
|
||||
|
||||
# stdout preserves info in CI logs in case of crash; history file is used for snapshot.
|
||||
printf "%s used=%s%% mem=%sMB goroutines=%s procs=[%s]\n" \
|
||||
"$timestamp" "$pct" "$memused_mb" "$goroutine_count" "$top_procs" | tee -a "$HISTORY_FILE"
|
||||
mkdir -p "$(dirname "$OOM_JUNIT_FILE")"
|
||||
cat > "$OOM_JUNIT_FILE" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuites tests="1" failures="1" errors="0" skipped="0" time="0">
|
||||
<testsuite name="memory_monitor" tests="1" failures="1" errors="0" skipped="0" time="0">
|
||||
<testcase classname="memory_monitor" name="OOM prevention" time="0">
|
||||
<failure type="OOM" message="OOM prevention threshold reached">Memory monitor terminated the test process at ${memory_pct}% memory before the runner OOM kill. See memory diagnostics artifacts.</failure>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
EOF
|
||||
}
|
||||
|
||||
# Build report.
|
||||
local report
|
||||
report="$(cat <<EOF
|
||||
Memory snapshot at $(date '+%Y-%m-%d %H:%M:%S') (usage ${pct}%)
|
||||
write_snapshot_report() {
|
||||
local memory_pct="$1"
|
||||
local optional_heap_profile_section="$2"
|
||||
|
||||
$(cat "$HISTORY_FILE")
|
||||
ensure_snapshot_dirs
|
||||
cat > "$SNAPSHOT_FILE" <<EOF
|
||||
Memory snapshot at $(date '+%Y-%m-%d %H:%M:%S') (usage ${memory_pct}%)
|
||||
|
||||
$(tail -120 "$SNAPSHOT_HISTORY_FILE")
|
||||
|
||||
--- Top Processes ---
|
||||
$(ps -eo pid,%mem,rss:10,comm --sort=-%mem | head -20)
|
||||
$(ps -eo pid,%mem,rss:10,comm --sort=-rss | head -10)
|
||||
|
||||
--- Memory Summary ---
|
||||
$(free -m)
|
||||
|
||||
$pprof_output
|
||||
|
||||
$goroutine_output
|
||||
$optional_heap_profile_section
|
||||
EOF
|
||||
)"
|
||||
}
|
||||
|
||||
# Print report to stdout when memory threshold is reached (only once per run).
|
||||
if [[ "$pct" -ge "$HIGH_MEMORY_THRESHOLD" ]] && [[ "$HEAP_PRINTED" == "false" ]]; then
|
||||
echo ""
|
||||
echo "$report"
|
||||
echo ""
|
||||
HEAP_PRINTED=true
|
||||
print_snapshot_status() {
|
||||
local now="$1"
|
||||
local status_line="$2"
|
||||
|
||||
if [[ $(( now - LAST_SNAPSHOT_PRINT_TIME )) -lt "$SNAPSHOT_PRINT_INTERVAL_SECONDS" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Write report to disk only if memory usage is at or above high water mark.
|
||||
if [[ "$pct" -ge "$HIGH_WATER_MARK" ]]; then
|
||||
echo "$report" > "$SNAPSHOT_FILE"
|
||||
HIGH_WATER_MARK="$pct"
|
||||
echo "$status_line"
|
||||
LAST_SNAPSHOT_PRINT_TIME="$now"
|
||||
}
|
||||
|
||||
capture_heap_profile() {
|
||||
local memory_pct="$1"
|
||||
local heap_profile_path_prefix heap_profile_summary
|
||||
|
||||
heap_profile_path_prefix="$HEAP_PROFILES_DIR/$(date '+%Y%m%d-%H%M%S')-${memory_pct}pct"
|
||||
mkdir -p "$(dirname "$heap_profile_path_prefix")"
|
||||
fetch_pprof "heap" "${heap_profile_path_prefix}.pb.gz" || true
|
||||
heap_profile_summary="$(print_heap_profile_summary "${heap_profile_path_prefix}.pb.gz")"
|
||||
|
||||
printf '\n%s\n' "$heap_profile_summary"
|
||||
}
|
||||
|
||||
should_capture_heap_profile() {
|
||||
local now="$1"
|
||||
local memory_pct="$2"
|
||||
local is_new_high="$3"
|
||||
local should_terminate_process_group="$4"
|
||||
|
||||
if [[ "$should_terminate_process_group" == "true" ]] && [[ "$HAS_CAPTURED_HEAP_PROFILE" == "false" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$is_new_high" != "true" ]] || [[ "$memory_pct" -lt "$HEAP_PROFILE_CAPTURE_THRESHOLD" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ $(( now - LAST_HEAP_PROFILE_CAPTURE_TIME )) -lt "$HEAP_PROFILE_REFRESH_INTERVAL_SECONDS" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
snapshot() {
|
||||
local memory_total_kb memory_available_kb memory_used_kb memory_used_mb memory_pct is_new_high should_terminate_process_group
|
||||
|
||||
memory_total_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo)"
|
||||
memory_available_kb="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)"
|
||||
memory_used_kb=$(( memory_total_kb - memory_available_kb ))
|
||||
memory_used_mb=$(( memory_used_kb / 1024 ))
|
||||
memory_pct=$(( memory_used_kb * 100 / memory_total_kb ))
|
||||
|
||||
# Get the top memory-heavy processes, format as "name (MB)".
|
||||
local top_processes
|
||||
top_processes="$(ps -eo rss,comm --sort=-rss | awk 'NR>1 && NR<=6 {printf "%s (%dMB), ", $2, $1/1024}' | sed 's/, $//')"
|
||||
|
||||
local timestamp
|
||||
timestamp="$(date '+%H:%M:%S')"
|
||||
local now
|
||||
now="$(date +%s)"
|
||||
|
||||
local status_line
|
||||
status_line="$(printf "%s used=%s%% mem=%sMB procs=[%s]" "$timestamp" "$memory_pct" "$memory_used_mb" "$top_processes")"
|
||||
ensure_snapshot_dirs
|
||||
echo "$status_line" >> "$SNAPSHOT_HISTORY_FILE"
|
||||
print_snapshot_status "$now" "$status_line"
|
||||
|
||||
is_new_high=false
|
||||
if [[ "$memory_pct" -gt "$MEMORY_HIGH_WATER_MARK" ]]; then
|
||||
is_new_high=true
|
||||
fi
|
||||
|
||||
should_terminate_process_group=false
|
||||
if [[ "$memory_pct" -ge "$OOM_TERMINATION_THRESHOLD" ]] && [[ "$OOM_TERMINATED" == "false" ]]; then
|
||||
should_terminate_process_group=true
|
||||
fi
|
||||
|
||||
if should_capture_heap_profile "$now" "$memory_pct" "$is_new_high" "$should_terminate_process_group"; then
|
||||
HEAP_PROFILE_SECTION="$(capture_heap_profile "$memory_pct")"
|
||||
LAST_HEAP_PROFILE_CAPTURE_TIME="$now"
|
||||
HAS_CAPTURED_HEAP_PROFILE=true
|
||||
fi
|
||||
|
||||
# Write the snapshot only at new memory highs so the final artifact represents
|
||||
# the worst observed point without emitting one file per sample.
|
||||
if [[ "$is_new_high" == "true" ]] || [[ ! -e "$SNAPSHOT_FILE" ]]; then
|
||||
write_snapshot_report "$memory_pct" "$HEAP_PROFILE_SECTION"
|
||||
MEMORY_HIGH_WATER_MARK="$memory_pct"
|
||||
fi
|
||||
|
||||
if [[ "$should_terminate_process_group" == "true" ]]; then
|
||||
OOM_TERMINATED=true
|
||||
write_oom_junit "$memory_pct"
|
||||
terminate_monitored_processes "$memory_pct"
|
||||
fi
|
||||
}
|
||||
|
||||
# Take snapshots every 30s until killed.
|
||||
init_snapshot_files
|
||||
|
||||
# Take snapshots until killed.
|
||||
while true; do
|
||||
snapshot
|
||||
sleep 30
|
||||
sleep "$SNAPSHOT_INTERVAL_SECONDS"
|
||||
done
|
||||
|
||||
@@ -19,10 +19,14 @@ fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Start monitor
|
||||
"$SCRIPT_DIR/memory_monitor.sh" /tmp/memory_snapshot.txt &
|
||||
MONITOR_PID=$!
|
||||
trap 'kill "$MONITOR_PID" 2>/dev/null' EXIT
|
||||
|
||||
# Run command
|
||||
"$@"
|
||||
setsid "$@" &
|
||||
COMMAND_PID=$!
|
||||
MONITORED_PROCESS_GROUP="$COMMAND_PID"
|
||||
|
||||
# Start monitor
|
||||
MONITORED_PROCESS_GROUP="$MONITORED_PROCESS_GROUP" "$SCRIPT_DIR/memory_monitor.sh" &
|
||||
MONITOR_PID=$!
|
||||
trap 'kill "$MONITOR_PID" 2>/dev/null || true' EXIT
|
||||
|
||||
wait "$COMMAND_PID"
|
||||
|
||||
Reference in New Issue
Block a user