diff --git a/.github/actions/post-test-reporting/action.yml b/.github/actions/post-test-reporting/action.yml index 438a94ebeb..e5a1990bd6 100644 --- a/.github/actions/post-test-reporting/action.yml +++ b/.github/actions/post-test-reporting/action.yml @@ -21,7 +21,7 @@ runs: - name: Print memory snapshot if: always() shell: bash - run: cat /tmp/memory_snapshot.txt || true + run: cat .testoutput/memory/memory-snapshot.txt || true - name: Generate crash report if: failure() @@ -122,3 +122,12 @@ runs: path: ./.testoutput/test-cluster-events.jsonl if-no-files-found: ignore retention-days: 14 + + - name: Upload memory diagnostics + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + if: ${{ !cancelled() }} + with: + name: memory-diagnostics--${{ github.run_id }}--${{ steps.get_job_id.outputs.job_id }}--${{ github.run_attempt }}--${{ inputs.artifact_name_suffix }} + path: ./.testoutput/memory/** + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 1afac909c5..634c1e8d30 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -495,7 +495,7 @@ jobs: - name: Print memory snapshot if: always() - run: cat /tmp/memory_snapshot.txt || true + run: cat .testoutput/memory/memory-snapshot.txt || true - name: Print current server logs if: always() diff --git a/develop/github/memory_monitor.sh b/develop/github/memory_monitor.sh index 751dfd1d07..135f9a9516 100755 --- a/develop/github/memory_monitor.sh +++ b/develop/github/memory_monitor.sh @@ -2,147 +2,247 @@ # # Memory Monitor # -# Takes periodic memory snapshots. Captures system memory stats, top processes, -# and for Go processes, heap profiles via pprof. +# 1. Snapshot status: +# Samples memory every SNAPSHOT_INTERVAL_SECONDS and writes every sample +# to SNAPSHOT_HISTORY_FILE. Logs status every SNAPSHOT_PRINT_INTERVAL_SECONDS +# and writes the highest-memory snapshot report to SNAPSHOT_FILE. +# +# 2. Profile capture: +# When usage crosses HEAP_PROFILE_CAPTURE_THRESHOLD, captures a heap +# profile in HEAP_PROFILES_DIR before running analysis. +# +# 3. OOM prevention: +# When usage crosses OOM_TERMINATION_THRESHOLD, reuses any previously +# captured profile, writes the latest snapshot and a synthetic JUnit +# report, then terminates the monitored test process group so post-test +# artifact upload can still run. If no profile has been captured yet, it +# captures one before terminating. # # Usage: -# ./memory_monitor.sh +# ./memory_monitor.sh # set -euo pipefail -if [[ $# -lt 1 ]]; then - echo "Usage: $0 " >&2 +if [[ $# -ne 0 ]]; then + echo "Usage: $0" >&2 exit 1 fi -SNAPSHOT_FILE="$1" -HISTORY_FILE="/tmp/memory_history.txt" -HIGH_MEMORY_THRESHOLD=95 -PPROF_HOST="${PPROF_HOST:-localhost:7000}" -HEAP_PRINTED=false -HIGH_WATER_MARK=0 +# Snapshot config. +readonly SNAPSHOT_DIR="${SNAPSHOT_DIR:-.testoutput/memory}" +# Sample every second so short OOM ramps still leave history, but print less +# often to keep CI logs readable. +readonly SNAPSHOT_INTERVAL_SECONDS="${SNAPSHOT_INTERVAL_SECONDS:-1}" +readonly SNAPSHOT_PRINT_INTERVAL_SECONDS="${SNAPSHOT_PRINT_INTERVAL_SECONDS:-30}" +readonly SNAPSHOT_FILE="${SNAPSHOT_FILE:-$SNAPSHOT_DIR/memory-snapshot.txt}" +readonly SNAPSHOT_HISTORY_FILE="${SNAPSHOT_HISTORY_FILE:-$SNAPSHOT_DIR/memory-history.txt}" -# Clear history on start -: > "$HISTORY_FILE" +# Heap profile config. +# Capture before the termination threshold so the diagnostic profile is usually +# available even if the runner kills the job before our termination path runs. +readonly HEAP_PROFILE_CAPTURE_THRESHOLD="${HEAP_PROFILE_CAPTURE_THRESHOLD:-90}" +readonly HEAP_PROFILE_REFRESH_INTERVAL_SECONDS="${HEAP_PROFILE_REFRESH_INTERVAL_SECONDS:-30}" +readonly HEAP_PROFILES_DIR="${HEAP_PROFILES_DIR:-$SNAPSHOT_DIR/heap-profiles}" +readonly PPROF_HOST="${PPROF_HOST:-localhost:7000}" + +# OOM prevention config. +# Terminate late enough to avoid masking near-finished tests, but before the +# runner OOM killer skips post-test artifact upload. +readonly OOM_TERMINATION_THRESHOLD="${OOM_TERMINATION_THRESHOLD:-99}" +readonly OOM_JUNIT_FILE="${OOM_JUNIT_FILE:-.testoutput/junit.oom.xml}" + +# State. +MEMORY_HIGH_WATER_MARK=0 +LAST_SNAPSHOT_PRINT_TIME=0 +LAST_HEAP_PROFILE_CAPTURE_TIME=0 +HEAP_PROFILE_SECTION="" +HAS_CAPTURED_HEAP_PROFILE=false +OOM_TERMINATED=false + +ensure_snapshot_dirs() { + mkdir -p "$(dirname "$SNAPSHOT_FILE")" "$(dirname "$SNAPSHOT_HISTORY_FILE")" +} + +init_snapshot_files() { + ensure_snapshot_dirs + : > "$SNAPSHOT_HISTORY_FILE" +} # Fetch a pprof profile and save to file -# Usage: fetch_pprof +# Usage: fetch_pprof # Returns 0 on success, 1 on failure fetch_pprof() { - local profile_type="$1" + local pprof_profile="$1" local output_file="$2" - curl -s --max-time 10 "http://${PPROF_HOST}/debug/pprof/${profile_type}" -o "$output_file" 2>/dev/null -} - -# Print pprof top analysis. -# Usage: pprof_top [extra_flags...] -pprof_top() { - local profile_file="$1" - local lines="$2" - shift 2 - - go tool pprof -top "$@" "$profile_file" 2>/dev/null | head -"$lines" || true -} - -# Print goroutine profile analysis. -print_goroutines() { - local tmp_file - tmp_file="$(mktemp)" - trap 'rm -f "$tmp_file"' RETURN - - if fetch_pprof "goroutine" "$tmp_file"; then - echo "=== top functions by goroutine count ===" - pprof_top "$tmp_file" 30 - fi -} - -# Extract goroutine count from pprof output ("... of N total"). -count_goroutines() { - sed -n 's/.*of \([0-9]*\) total.*/\1/p' | head -1 || echo '?' + curl -s --max-time 10 "http://${PPROF_HOST}/debug/pprof/${pprof_profile}" -o "$output_file" 2>/dev/null } # Print heap profile analysis. -print_heap() { - local tmp_file - tmp_file="$(mktemp)" - trap 'rm -f "$tmp_file"' RETURN +print_heap_profile_summary() { + local heap_profile_file="$1" echo "--- Go Heap Profile ---" - if fetch_pprof "heap" "$tmp_file"; then + if [[ -s "$heap_profile_file" ]]; then echo "=== inuse_space (what's currently held) ===" - pprof_top "$tmp_file" 30 -inuse_space - echo "" - echo "=== alloc_space (total allocations) ===" - pprof_top "$tmp_file" 30 -alloc_space - echo "" - echo "=== alloc_objects (total objects allocated) ===" - pprof_top "$tmp_file" 30 -alloc_objects + # Keep the artifact focused on retained memory; allocation totals are noisy + # for this CI OOM investigation. + go tool pprof -top -inuse_space "$heap_profile_file" 2>/dev/null | head -15 || true else - echo "(pprof endpoint not available)" + echo "(heap profile not available)" fi } -snapshot() { - local memtotal_kb memavail_kb memused_kb memused_mb pct - memtotal_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo)" - memavail_kb="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)" - memused_kb=$(( memtotal_kb - memavail_kb )) - memused_mb=$(( memused_kb / 1024 )) - pct=$(( memused_kb * 100 / memtotal_kb )) +terminate_monitored_processes() { + local memory_pct="$1" - # Collect pprof data once per tick. - local goroutine_output goroutine_count pprof_output - goroutine_output="$(print_goroutines)" - goroutine_count="$(count_goroutines <<< "$goroutine_output")" - pprof_output="$(print_heap)" + if [[ -n "${MONITORED_PROCESS_GROUP:-}" ]]; then + echo "Terminating monitored process group ${MONITORED_PROCESS_GROUP} at ${memory_pct}% memory to preserve diagnostics artifacts." + kill -TERM "-$MONITORED_PROCESS_GROUP" 2>/dev/null || true + return + fi - # Get processes with >=1% memory, format as "name (MB)" - local top_procs - top_procs="$(ps -eo %mem,rss,comm --sort=-%mem | awk 'NR>1 && $1>=1.0 {printf "%s (%dMB), ", $3, $2/1024}' | sed 's/, $//')" + echo "No monitored process group set at ${memory_pct}% memory; leaving processes running." +} - local timestamp - timestamp="$(date '+%H:%M:%S')" +write_oom_junit() { + local memory_pct="$1" - # stdout preserves info in CI logs in case of crash; history file is used for snapshot. - printf "%s used=%s%% mem=%sMB goroutines=%s procs=[%s]\n" \ - "$timestamp" "$pct" "$memused_mb" "$goroutine_count" "$top_procs" | tee -a "$HISTORY_FILE" + mkdir -p "$(dirname "$OOM_JUNIT_FILE")" + cat > "$OOM_JUNIT_FILE" < + + + + Memory monitor terminated the test process at ${memory_pct}% memory before the runner OOM kill. See memory diagnostics artifacts. + + + +EOF +} - # Build report. - local report - report="$(cat < "$SNAPSHOT_FILE" < "$SNAPSHOT_FILE" - HIGH_WATER_MARK="$pct" + echo "$status_line" + LAST_SNAPSHOT_PRINT_TIME="$now" +} + +capture_heap_profile() { + local memory_pct="$1" + local heap_profile_path_prefix heap_profile_summary + + heap_profile_path_prefix="$HEAP_PROFILES_DIR/$(date '+%Y%m%d-%H%M%S')-${memory_pct}pct" + mkdir -p "$(dirname "$heap_profile_path_prefix")" + fetch_pprof "heap" "${heap_profile_path_prefix}.pb.gz" || true + heap_profile_summary="$(print_heap_profile_summary "${heap_profile_path_prefix}.pb.gz")" + + printf '\n%s\n' "$heap_profile_summary" +} + +should_capture_heap_profile() { + local now="$1" + local memory_pct="$2" + local is_new_high="$3" + local should_terminate_process_group="$4" + + if [[ "$should_terminate_process_group" == "true" ]] && [[ "$HAS_CAPTURED_HEAP_PROFILE" == "false" ]]; then + return 0 + fi + + if [[ "$is_new_high" != "true" ]] || [[ "$memory_pct" -lt "$HEAP_PROFILE_CAPTURE_THRESHOLD" ]]; then + return 1 + fi + + if [[ $(( now - LAST_HEAP_PROFILE_CAPTURE_TIME )) -lt "$HEAP_PROFILE_REFRESH_INTERVAL_SECONDS" ]]; then + return 1 + fi + + return 0 +} + +snapshot() { + local memory_total_kb memory_available_kb memory_used_kb memory_used_mb memory_pct is_new_high should_terminate_process_group + + memory_total_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo)" + memory_available_kb="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)" + memory_used_kb=$(( memory_total_kb - memory_available_kb )) + memory_used_mb=$(( memory_used_kb / 1024 )) + memory_pct=$(( memory_used_kb * 100 / memory_total_kb )) + + # Get the top memory-heavy processes, format as "name (MB)". + local top_processes + top_processes="$(ps -eo rss,comm --sort=-rss | awk 'NR>1 && NR<=6 {printf "%s (%dMB), ", $2, $1/1024}' | sed 's/, $//')" + + local timestamp + timestamp="$(date '+%H:%M:%S')" + local now + now="$(date +%s)" + + local status_line + status_line="$(printf "%s used=%s%% mem=%sMB procs=[%s]" "$timestamp" "$memory_pct" "$memory_used_mb" "$top_processes")" + ensure_snapshot_dirs + echo "$status_line" >> "$SNAPSHOT_HISTORY_FILE" + print_snapshot_status "$now" "$status_line" + + is_new_high=false + if [[ "$memory_pct" -gt "$MEMORY_HIGH_WATER_MARK" ]]; then + is_new_high=true + fi + + should_terminate_process_group=false + if [[ "$memory_pct" -ge "$OOM_TERMINATION_THRESHOLD" ]] && [[ "$OOM_TERMINATED" == "false" ]]; then + should_terminate_process_group=true + fi + + if should_capture_heap_profile "$now" "$memory_pct" "$is_new_high" "$should_terminate_process_group"; then + HEAP_PROFILE_SECTION="$(capture_heap_profile "$memory_pct")" + LAST_HEAP_PROFILE_CAPTURE_TIME="$now" + HAS_CAPTURED_HEAP_PROFILE=true + fi + + # Write the snapshot only at new memory highs so the final artifact represents + # the worst observed point without emitting one file per sample. + if [[ "$is_new_high" == "true" ]] || [[ ! -e "$SNAPSHOT_FILE" ]]; then + write_snapshot_report "$memory_pct" "$HEAP_PROFILE_SECTION" + MEMORY_HIGH_WATER_MARK="$memory_pct" + fi + + if [[ "$should_terminate_process_group" == "true" ]]; then + OOM_TERMINATED=true + write_oom_junit "$memory_pct" + terminate_monitored_processes "$memory_pct" fi } -# Take snapshots every 30s until killed. +init_snapshot_files + +# Take snapshots until killed. while true; do snapshot - sleep 30 + sleep "$SNAPSHOT_INTERVAL_SECONDS" done diff --git a/develop/github/monitor_test.sh b/develop/github/monitor_test.sh index 6c0f773b89..88e7d097f4 100755 --- a/develop/github/monitor_test.sh +++ b/develop/github/monitor_test.sh @@ -19,10 +19,14 @@ fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Start monitor -"$SCRIPT_DIR/memory_monitor.sh" /tmp/memory_snapshot.txt & -MONITOR_PID=$! -trap 'kill "$MONITOR_PID" 2>/dev/null' EXIT - # Run command -"$@" +setsid "$@" & +COMMAND_PID=$! +MONITORED_PROCESS_GROUP="$COMMAND_PID" + +# Start monitor +MONITORED_PROCESS_GROUP="$MONITORED_PROCESS_GROUP" "$SCRIPT_DIR/memory_monitor.sh" & +MONITOR_PID=$! +trap 'kill "$MONITOR_PID" 2>/dev/null || true' EXIT + +wait "$COMMAND_PID"