Capture memory monitor report before OOM crash (#9079)

## What changed?

Follow-up to https://github.com/temporalio/temporal/pull/9027

1. Print memory stats to both file and stdout (new!)
2. Print Go heap report once a memory threshold is reached (ie close to
OOM kill)
3. Add `alloc_objects` to Go heap report


See
[example](https://github.com/temporalio/temporal/actions/runs/21153910359/job/60835302610?pr=9079#step:9:89):
<img width="1328" height="1197" alt="Screenshot 2026-01-19 at 4 23
28 PM"
src="https://github.com/user-attachments/assets/fe910197-d813-4f63-8907-797c5f11bc90"
/>
(note that Go test output is buffered so they all appear mostly
sequentially)

## Why?

Gain ability to capture memory usage and Go heap report **before an OOM
kill**.

## How did you test it?
- [ ] built
- [ ] run locally and tested manually
- [x] covered by existing tests
- [ ] added new unit test(s)
- [ ] added new functional test(s)

develop/github/memory_monitor.sh

@@ -17,11 +17,68 @@ fi
 
 SNAPSHOT_FILE="$1"
 HISTORY_FILE="/tmp/memory_history.txt"
+HIGH_MEMORY_THRESHOLD=95
+PPROF_HOST="${PPROF_HOST:-localhost:7000}"
+HEAP_PRINTED=false
 
 # Clear history on start
 : > "$HISTORY_FILE"
 
-write_snapshot() {
+# Fetch a pprof profile and save to file
+# Usage: fetch_pprof <profile_type> <output_file>
+# Returns 0 on success, 1 on failure
+fetch_pprof() {
+  local profile_type="$1"
+  local output_file="$2"
+  curl -s --max-time 10 "http://${PPROF_HOST}/debug/pprof/${profile_type}" -o "$output_file" 2>/dev/null
+}
+
+# Print heap analysis from `go tool pprof`.
+# Usage: print_heap_analysis <profile_file> <mode> <lines>
+# mode: inuse_space, alloc_space, inuse_objects, alloc_objects
+print_heap_analysis() {
+  local profile_file="$1"
+  local mode="$2"
+  local lines="${3:-40}"
+
+  go tool pprof -top "-${mode}" "$profile_file" 2>/dev/null | head -"$lines" || true
+}
+
+# Get goroutine count from pprof.
+print_goroutine_count() {
+  local tmp_file
+  tmp_file="$(mktemp)"
+  trap 'rm -f "$tmp_file"' RETURN
+
+  local count="?"
+  if curl -s --max-time 5 "http://${PPROF_HOST}/debug/pprof/goroutine?debug=1" -o "$tmp_file" 2>/dev/null; then
+    count="$(head -1 "$tmp_file" | grep -o '[0-9]*' || echo '?')"
+  fi
+  echo "$count"
+}
+
+# Print pprof heap analysis.
+print_pprof_analysis() {
+  local tmp_file
+  tmp_file="$(mktemp)"
+  trap 'rm -f "$tmp_file"' RETURN
+
+  echo "--- Go Heap Profile ---"
+  if fetch_pprof "heap" "$tmp_file"; then
+    echo "=== inuse_space (what's currently held) ==="
+    print_heap_analysis "$tmp_file" "inuse_space" 30
+    echo ""
+    echo "=== alloc_space (total allocations) ==="
+    print_heap_analysis "$tmp_file" "alloc_space" 30
+    echo ""
+    echo "=== alloc_objects (total objects allocated) ==="
+    print_heap_analysis "$tmp_file" "alloc_objects" 30
+  else
+    echo "(pprof endpoint not available)"
+  fi
+}
+
+snapshot() {
   local memtotal_kb memavail_kb memused_kb memused_mb pct
   memtotal_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo)"
   memavail_kb="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)"
@@ -29,10 +86,8 @@ write_snapshot() {
   memused_mb=$(( memused_kb / 1024 ))
   pct=$(( memused_kb * 100 / memtotal_kb ))
 
-  local goroutines="?"
-  if curl -s --max-time 5 'http://localhost:7000/debug/pprof/goroutine?debug=1' -o /tmp/goroutine.out 2>/dev/null; then
-    goroutines="$(head -1 /tmp/goroutine.out | grep -o '[0-9]*' || echo '?')"
-  fi
+  local goroutines
+  goroutines="$(print_goroutine_count)"
 
   # Get processes with >=1% memory, format as "name (MB)"
   local top_procs
@@ -41,17 +96,29 @@ write_snapshot() {
   local timestamp
   timestamp="$(date '+%H:%M:%S')"
 
-  # Append to history
-  echo "$timestamp $pct $memused_mb $goroutines $top_procs" >> "$HISTORY_FILE"
+  # stdout preserves info in CI logs in case of crash; history file is used for snapshot.
+  printf "%s used=%s%% mem=%sMB goroutines=%s procs=[%s]\n" \
+    "$timestamp" "$pct" "$memused_mb" "$goroutines" "$top_procs" | tee -a "$HISTORY_FILE"
 
+  # Collect pprof analysis once per tick.
+  local pprof_output
+  pprof_output="$(print_pprof_analysis)"
+
+  # If memory threshold was reached, print Go heap details to stdout. But only once per run.
+  if [[ "$pct" -ge "$HIGH_MEMORY_THRESHOLD" ]] && [[ "$HEAP_PRINTED" == "false" ]]; then
+    echo ""
+    echo "=== HIGH MEMORY WARNING: ${pct}% used (threshold: ${HIGH_MEMORY_THRESHOLD}%) ==="
+    echo "$pprof_output"
+    echo "=== END HIGH MEMORY WARNING ==="
+    echo ""
+    HEAP_PRINTED=true
+  fi
+
+  # Write snapshot to disk.
   {
     echo "Memory snapshot at $(date '+%Y-%m-%d %H:%M:%S')"
     echo ""
-    echo "Time      Used(%)  Used(MB)  Goroutines  Top Processes"
-    echo "---------------------------------------------------------------"
-    while read -r t p mb g procs; do
-      printf "%-10s %3s%%     %5s   %7s     %s\n" "$t" "$p" "$mb" "$g" "$procs"
-    done < "$HISTORY_FILE"
+    cat "$HISTORY_FILE"
     echo ""
     echo "--- Top Processes ---"
     ps -eo pid,%mem,rss:10,comm --sort=-%mem | head -20
@@ -59,19 +126,12 @@ write_snapshot() {
     echo "--- Memory Summary ---"
     free -m
     echo ""
-    echo "--- Go Heap Profile ---"
-    if curl -s --max-time 10 "http://localhost:7000/debug/pprof/heap" -o /tmp/heap.out 2>/dev/null; then
-      go tool pprof -top -inuse_space /tmp/heap.out 2>/dev/null | head -60 || true
-      echo ""
-      go tool pprof -top -alloc_space /tmp/heap.out 2>/dev/null | head -60 || true
-    else
-      echo "(pprof endpoint not available)"
-    fi
+    echo "$pprof_output"
   } > "$SNAPSHOT_FILE"
 }
 
-# Take snapshots every 30s until killed
+# Take snapshots every 30s until killed.
 while true; do
-  write_snapshot
+  snapshot
   sleep 30
 done