mirror of
https://github.com/temporalio/temporal.git
synced 2026-08-31 02:51:51 -07:00
Memory watcher for functional tests (#9027)
## What changed? Added script to take memory report snapshots during test execution and print report after test success/failure. ## Why? Gain insights into memory usage; especially when job is OOM killed. ## How did you test it? see example: https://github.com/temporalio/temporal/actions/runs/21011468266/job/60407413329?pr=9027#step:9:15 <details><summary>Example</summary> <p> ``` Memory snapshot at 2026-01-14 22:16:53 Time Used(%) Used(MB) Goroutines Top Processes --------------------------------------------------------------- 22:06:03 16% 5347 ? java (2468MB), java (1322MB) 22:06:33 20% 6460 ? java (2450MB), java (1322MB), compile (376MB) 22:07:03 16% 5418 ? java (2430MB), java (1323MB) 22:07:33 16% 5401 2781 java (2531MB), java (1333MB) 22:08:06 17% 5702 5861 java (2493MB), java (1339MB), xdc.test (649MB) 22:08:36 18% 6034 5971 java (2528MB), java (1346MB), xdc.test (936MB) 22:09:07 20% 6455 6149 java (2539MB), java (1350MB), xdc.test (1314MB) 22:09:38 26% 8416 45668 xdc.test (3281MB), java (2535MB), java (1375MB) 22:10:10 33% 10596 43034 xdc.test (5657MB), java (2560MB), java (1405MB) 22:10:42 39% 12806 39209 xdc.test (7449MB), java (2551MB), java (1413MB) 22:11:13 42% 13580 33372 xdc.test (8166MB), java (2556MB), java (1413MB) 22:11:44 44% 14196 11067 xdc.test (8168MB), java (2548MB), java (1413MB) 22:12:15 44% 14171 11342 xdc.test (8653MB), java (2555MB), java (1415MB) 22:12:46 44% 14233 11724 xdc.test (8682MB), java (2546MB), java (1421MB) 22:13:17 45% 14536 10186 xdc.test (8858MB), java (2556MB), java (1427MB) 22:13:48 45% 14648 10392 xdc.test (8961MB), java (2555MB), java (1427MB) 22:14:19 45% 14710 10935 xdc.test (9027MB), java (2552MB), java (1428MB) 22:14:49 45% 14716 11060 xdc.test (9020MB), java (2557MB), java (1428MB) 22:15:20 45% 14702 11312 xdc.test (8925MB), java (2557MB), java (1428MB) 22:15:51 46% 14835 11571 xdc.test (9088MB), java (2545MB), java (1429MB) 22:16:22 46% 14839 11964 xdc.test (9082MB), java (2556MB), java (1429MB) 22:16:53 18% 5911 5792 java (2547MB), java (1434MB), xdc.test (628MB) --- Top Processes --- PID %MEM RSS COMMAND 2550 7.9 2608560 java 2679 4.4 1468908 java 23586 1.9 643644 xdc.test 1810 0.4 155452 provjobd 2094 0.4 142716 Runner.Worker 2079 0.3 113336 Runner.Listener 23517 0.3 99448 go 1161 0.2 78640 dockerd 833 0.1 51476 php-fpm8.3 910 0.1 51092 containerd 270 0.1 43536 multipathd 1417 0.1 36216 python3 865 0.1 35444 python3 3362 0.0 22220 go 1184 0.0 21448 php-fpm8.3 1185 0.0 21448 php-fpm8.3 831 0.0 20332 networkd-dispat 1987 0.0 19000 hca 203 0.0 16660 systemd-journal --- Memory Summary --- total used free shared buff/cache available Mem: 32098 5914 17968 51 8738 26183 Swap: 0 0 0 --- Go Heap Profile --- File: xdc.test Build ID: 2f2150795a26dac09836852e7ea9c9cf9f89fc14 Type: inuse_space Time: 2026-01-14 22:16:53 UTC Showing nodes accounting for 37251.65kB, 100% of 37251.65kB total flat flat% sum% cum cum% 6002.03kB 16.11% 16.11% 6002.03kB 16.11% go.temporal.io/sdk/internal.newLocalActivityTunnel (inline) 2565kB 6.89% 23.00% 2565kB 6.89% runtime.allocm 2564.16kB 6.88% 29.88% 3635.80kB 9.76% google.golang.org/protobuf/internal/filedesc.(*Message).unmarshalFull 2384.64kB 6.40% 36.28% 2384.64kB 6.40% go.temporal.io/server/common/aggregate.NewMovingWindowAvgImpl (inline) 2048.35kB 5.50% 41.78% 2048.35kB 5.50% github.com/aws/aws-sdk-go/aws/endpoints.init 1537.50kB 4.13% 45.91% 1537.50kB 4.13% go.uber.org/zap/internal/bufferpool.init.NewPool.func1 1536.66kB 4.13% 50.03% 1536.66kB 4.13% runtime.malg 1072.75kB 2.88% 52.91% 1072.75kB 2.88% go.temporal.io/sdk/internal.newBaseWorker 1071.64kB 2.88% 55.79% 1071.64kB 2.88% google.golang.org/protobuf/internal/strs.(*Builder).grow (inline) 1045.40kB 2.81% 58.60% 1045.40kB 2.81% github.com/gocql/gocql.(*Session).scanColumnMetadataSystem 1024.11kB 2.75% 61.35% 1024.11kB 2.75% time.newTimer 1024.09kB 2.75% 64.09% 1024.09kB 2.75% go.temporal.io/server/common/tasks.(*SequentialScheduler[go.shape.interface { Abort(); Ack(); Cancel(); Execute() error; HandleErr(error) error; IsRetryableError(error) bool; MarkPoisonPill() error; Nack(error); QueueID() interface {}; ReplicationTask() *go.temporal.io/server/api/replication/v1.ReplicationTask; Reschedule(); RetryPolicy() go.temporal.io/server/common/backoff.RetryPolicy; SourceClusterName() string; State() go.temporal.io/server/common/tasks.State; TaskCreationTime() time.Time; TaskID() int64 }]).pollTaskQueue 532.26kB 1.43% 65.52% 532.26kB 1.43% google.golang.org/protobuf/reflect/protoregistry.(*Files).RegisterFile.func2 528.17kB 1.42% 66.94% 528.17kB 1.42% bufio.NewReaderSize (inline) 528.17kB 1.42% 68.36% 528.17kB 1.42% google.golang.org/grpc/internal/transport.newBufWriter (inline) 516.76kB 1.39% 69.75% 516.76kB 1.39% runtime.procresize 514kB 1.38% 71.13% 514kB 1.38% github.com/gocql/gocql/internal/streams.New (inline) 513.50kB 1.38% 72.50% 513.50kB 1.38% go.temporal.io/server/api/history/v1.init 512.50kB 1.38% 73.88% 7680.97kB 20.62% go.uber.org/fx/internal/lifecycle.(*Lifecycle).Start 512.44kB 1.38% 75.26% 512.44kB 1.38% go/token.init.0 512.25kB 1.38% 76.63% 512.25kB 1.38% runtime.gcBgMarkWorker 512.19kB 1.37% 78.01% 512.19kB 1.37% github.com/gocql/gocql.(*Conn).executeQuery 512.17kB 1.37% 79.38% 512.17kB 1.37% google.golang.org/grpc/metadata.FromOutgoingContext 512.17kB 1.37% 80.76% 512.17kB 1.37% google.golang.org/grpc/metadata.MD.Set (inline) 512.10kB 1.37% 82.13% 512.10kB 1.37% context.(*cancelCtx).propagateCancel 512.10kB 1.37% 83.51% 512.10kB 1.37% go.uber.org/fx.moduleOption.apply 512.09kB 1.37% 84.88% 512.09kB 1.37% internal/abi.NewName 512.07kB 1.37% 86.25% 512.07kB 1.37% go.uber.org/dig.(*Scope).newGraphNode 512.06kB 1.37% 87.63% 512.06kB 1.37% google.golang.org/protobuf/internal/impl.fieldInfoForScalar 512.05kB 1.37% 89.00% 512.05kB 1.37% go.temporal.io/server/common/goro.NewAdaptivePool 512.05kB 1.37% 90.38% 512.05kB 1.37% go.temporal.io/server/common/nexus.(*EndpointRegistryImpl).setEnabled 512.05kB 1.37% 91.75% 512.05kB 1.37% google.golang.org/grpc/internal/transport.newWriteQuota (inline) 512.05kB 1.37% 93.13% 512.05kB 1.37% go.temporal.io/sdk/internal.(*baseWorker).runEagerTaskDispatcher 512.02kB 1.37% 94.50% 512.02kB 1.37% context.WithValue 512.02kB 1.37% 95.88% 512.02kB 1.37% encoding/json.typeFields 512.02kB 1.37% 97.25% 512.02kB 1.37% github.com/gocql/gocql.(*ring).addHostIfMissing 512.02kB 1.37% 98.63% 512.02kB 1.37% github.com/google/uuid.UUID.String (inline) 512.02kB 1.37% 100% 512.02kB 1.37% go.uber.org/dig.newParam 0 0% 100% 512.10kB 1.37% context.WithCancel 0 0% 100% 512.10kB 1.37% context.withCancel (inline) 0 0% 100% 512.02kB 1.37% encoding/json.(*Decoder).Decode 0 0% 100% 512.02kB 1.37% encoding/json.(*decodeState).object 0 0% 100% 512.02kB 1.37% encoding/json.(*decodeState).unmarshal 0 0% 100% 512.02kB 1.37% encoding/json.(*decodeState).value 0 0% 100% 512.02kB 1.37% encoding/json.cachedTypeFields 0 0% 100% 512.02kB 1.37% encoding/json.newStructEncoder (inline) 0 0% 100% 512.02kB 1.37% encoding/json.newTypeEncoder 0 0% 100% 512.02kB 1.37% encoding/json.typeEncoder 0 0% 100% 512.02kB 1.37% encoding/json.typeEncoder.func1 0 0% 100% 1557.42kB 4.18% github.com/gocql/gocql.(*ClusterConfig).CreateSession (inline) 0 0% 100% 512.19kB 1.37% github.com/gocql/gocql.(*Query).Iter 0 0% 100% 512.19kB 1.37% github.com/gocql/gocql.(*Query).MapScanCAS 0 0% 100% 512.19kB 1.37% github.com/gocql/gocql.(*Query).execute 0 0% 100% 1045.40kB 2.81% github.com/gocql/gocql.(*Session).KeyspaceMetadata File: xdc.test Build ID: 2f2150795a26dac09836852e7ea9c9cf9f89fc14 Type: alloc_space Time: 2026-01-14 22:16:53 UTC Showing nodes accounting for 99.45MB, 80.53% of 123.49MB total Dropped 305 nodes (cum <= 0.62MB) flat flat% sum% cum cum% 51.44MB 41.65% 41.65% 51.44MB 41.65% regexp.(*bitState).reset 5.86MB 4.75% 46.40% 5.86MB 4.75% go.temporal.io/sdk/internal.newLocalActivityTunnel (inline) 3.53MB 2.85% 49.26% 4.59MB 3.72% compress/flate.NewWriter (inline) 2.50MB 2.03% 51.28% 2.50MB 2.03% runtime.allocm 2.50MB 2.03% 53.31% 3.55MB 2.88% google.golang.org/protobuf/internal/filedesc.(*Message).unmarshalFull 2.33MB 1.89% 55.20% 2.33MB 1.89% go.temporal.io/server/common/aggregate.NewMovingWindowAvgImpl (inline) 2MB 1.62% 56.82% 2MB 1.62% go.uber.org/zap/internal/bufferpool.init.NewPool.func1 2MB 1.62% 58.44% 2MB 1.62% github.com/aws/aws-sdk-go/aws/endpoints.init 1.52MB 1.23% 59.67% 1.52MB 1.23% github.com/gocql/gocql.(*Session).scanColumnMetadataSystem 1.51MB 1.22% 60.89% 1.51MB 1.22% github.com/grpc-ecosystem/grpc-gateway/v2/runtime.(*ServeMux).Handle 1.50MB 1.22% 62.11% 1.50MB 1.22% runtime.malg 1.50MB 1.21% 63.32% 1.50MB 1.21% fmt.init.func1 1.50MB 1.21% 64.54% 1.50MB 1.21% github.com/gocql/gocql.newFramer (inline) 1.50MB 1.21% 65.75% 2MB 1.62% go.uber.org/dig.getParamOrder 1.07MB 0.86% 66.62% 1.07MB 0.86% compress/flate.(*compressor).initDeflate (inline) 1.05MB 0.85% 67.46% 1.05MB 0.85% go.temporal.io/sdk/internal.newBaseWorker 1.05MB 0.85% 68.31% 1.05MB 0.85% google.golang.org/protobuf/internal/strs.(*Builder).grow (inline) 1.03MB 0.84% 69.15% 1.03MB 0.84% compress/flate.(*dictDecoder).init (inline) 1.02MB 0.83% 69.97% 1.02MB 0.83% google.golang.org/protobuf/reflect/protoregistry.(*Files).RegisterFile.func2 1MB 0.81% 70.79% 1MB 0.81% github.com/gocql/gocql/internal/streams.New (inline) 1MB 0.81% 71.60% 2MB 1.62% go.uber.org/dig.(*Scope).findAndValidateResults 1MB 0.81% 72.41% 1MB 0.81% context.(*cancelCtx).propagateCancel 1MB 0.81% 73.22% 1MB 0.81% time.newTimer 1MB 0.81% 74.03% 1MB 0.81% go.temporal.io/server/common/tasks.(*SequentialScheduler[go.shape.interface { Abort(); Ack(); Cancel(); Execute() error; HandleErr(error) error; IsRetryableError(error) bool; MarkPoisonPill() error; Nack(error); QueueID() interface {}; ReplicationTask() *go.temporal.io/server/api/replication/v1.ReplicationTask; Reschedule(); RetryPolicy() go.temporal.io/server/common/backoff.RetryPolicy; SourceClusterName() string; State() go.temporal.io/server/common/tasks.State; TaskCreationTime() time.Time; TaskID() int64 }]).pollTaskQueue 1MB 0.81% 74.84% 7MB 5.67% go.uber.org/dig.(*Scope).provide 1MB 0.81% 75.65% 1MB 0.81% container/list.New (inline) 1MB 0.81% 76.46% 2MB 1.62% go.uber.org/dig.newParam 0.52MB 0.42% 76.88% 1.54MB 1.25% google.golang.org/protobuf/reflect/protoregistry.(*Files).RegisterFile 0.51MB 0.41% 77.29% 1.54MB 1.24% io.ReadAll 0.50MB 0.41% 77.69% 8.50MB 6.88% go.uber.org/fx/internal/lifecycle.(*Lifecycle).Start 0.50MB 0.41% 78.10% 2MB 1.62% github.com/gocql/gocql.(*Conn).executeQuery 0.50MB 0.41% 78.50% 1.50MB 1.22% go.temporal.io/server/common/log.(*zapLogger).cloneWithTags 0.50MB 0.41% 78.91% 10.98MB 8.89% go.uber.org/dig.paramObject.Build 0.50MB 0.4% 79.31% 37.04MB 29.99% go.uber.org/fx/internal/fxreflect.CallerStack 0.50MB 0.4% 79.72% 2.50MB 2.02% go.uber.org/dig.(*graphHolder).EdgesFrom 0.50MB 0.4% 80.12% 7.50MB 6.07% go.uber.org/dig.(*Scope).Provide 0.50MB 0.4% 80.53% 1MB 0.81% go.uber.org/dig.(*Scope).getAllProviders (inline) 0 0% 80.53% 1.07MB 0.86% compress/flate.(*compressor).init 0 0% 80.53% 1.03MB 0.84% compress/flate.NewReader 0 0% 80.53% 1.03MB 0.84% compress/gzip.(*Reader).Reset 0 0% 80.53% 1.03MB 0.84% compress/gzip.(*Reader).readHeader 0 0% 80.53% 4.59MB 3.72% compress/gzip.(*Writer).Write 0 0% 80.53% 1.03MB 0.84% compress/gzip.NewReader (inline) 0 0% 80.53% 1MB 0.81% context.WithCancel 0 0% 80.53% 1MB 0.81% context.withCancel (inline) 0 0% 80.53% 1MB 0.81% fmt.(*pp).doPrintf 0 0% 80.53% 1.02MB 0.82% fmt.(*pp).handleMethods 0 0% 80.53% 1.52MB 1.23% fmt.(*pp).printArg 0 0% 80.53% 2MB 1.62% fmt.Fprintf 0 0% 80.53% 1MB 0.81% fmt.Sprintf 0 0% 80.53% 1.50MB 1.21% fmt.newPrinter 0 0% 80.53% 4.02MB 3.26% github.com/gocql/gocql.(*ClusterConfig).CreateSession (inline) 0 0% 80.53% 2MB 1.62% github.com/gocql/gocql.(*Conn).recv ``` </p> </details> --------- Co-authored-by: David Reiss <dnr@dnr.im>
This commit is contained in:
Stephan Behnke
77
develop/github/memory_monitor.sh
Executable file
77
develop/github/memory_monitor.sh
Executable file
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Memory Monitor
|
||||
#
|
||||
# Takes periodic memory snapshots. Captures system memory stats, top processes,
|
||||
# and for Go processes, heap profiles via pprof.
|
||||
#
|
||||
# Usage:
|
||||
# ./memory_monitor.sh <snapshot-file>
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
echo "Usage: $0 <snapshot-file>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SNAPSHOT_FILE="$1"
|
||||
HISTORY_FILE="/tmp/memory_history.txt"
|
||||
|
||||
# Clear history on start
|
||||
: > "$HISTORY_FILE"
|
||||
|
||||
write_snapshot() {
|
||||
local memtotal_kb memavail_kb memused_kb memused_mb pct
|
||||
memtotal_kb="$(awk '/MemTotal/ {print $2}' /proc/meminfo)"
|
||||
memavail_kb="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)"
|
||||
memused_kb=$(( memtotal_kb - memavail_kb ))
|
||||
memused_mb=$(( memused_kb / 1024 ))
|
||||
pct=$(( memused_kb * 100 / memtotal_kb ))
|
||||
|
||||
local goroutines="?"
|
||||
if curl -s --max-time 5 'http://localhost:7000/debug/pprof/goroutine?debug=1' -o /tmp/goroutine.out 2>/dev/null; then
|
||||
goroutines="$(head -1 /tmp/goroutine.out | grep -o '[0-9]*' || echo '?')"
|
||||
fi
|
||||
|
||||
# Get processes with >=1% memory, format as "name (MB)"
|
||||
local top_procs
|
||||
top_procs="$(ps -eo %mem,rss,comm --sort=-%mem | awk 'NR>1 && $1>=1.0 {printf "%s (%dMB), ", $3, $2/1024}' | sed 's/, $//')"
|
||||
|
||||
local timestamp
|
||||
timestamp="$(date '+%H:%M:%S')"
|
||||
|
||||
# Append to history
|
||||
echo "$timestamp $pct $memused_mb $goroutines $top_procs" >> "$HISTORY_FILE"
|
||||
|
||||
{
|
||||
echo "Memory snapshot at $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
echo "Time Used(%) Used(MB) Goroutines Top Processes"
|
||||
echo "---------------------------------------------------------------"
|
||||
while read -r t p mb g procs; do
|
||||
printf "%-10s %3s%% %5s %7s %s\n" "$t" "$p" "$mb" "$g" "$procs"
|
||||
done < "$HISTORY_FILE"
|
||||
echo ""
|
||||
echo "--- Top Processes ---"
|
||||
ps -eo pid,%mem,rss:10,comm --sort=-%mem | head -20
|
||||
echo ""
|
||||
echo "--- Memory Summary ---"
|
||||
free -m
|
||||
echo ""
|
||||
echo "--- Go Heap Profile ---"
|
||||
if curl -s --max-time 10 "http://localhost:7000/debug/pprof/heap" -o /tmp/heap.out 2>/dev/null; then
|
||||
go tool pprof -top -inuse_space /tmp/heap.out 2>/dev/null | head -60 || true
|
||||
echo ""
|
||||
go tool pprof -top -alloc_space /tmp/heap.out 2>/dev/null | head -60 || true
|
||||
else
|
||||
echo "(pprof endpoint not available)"
|
||||
fi
|
||||
} > "$SNAPSHOT_FILE"
|
||||
}
|
||||
|
||||
# Take snapshots every 30s until killed
|
||||
while true; do
|
||||
write_snapshot
|
||||
sleep 30
|
||||
done
|
||||
28
develop/github/monitor_test.sh
Executable file
28
develop/github/monitor_test.sh
Executable file
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Monitor Test
|
||||
#
|
||||
# Runs the given command with background monitoring.
|
||||
#
|
||||
# Usage:
|
||||
# ./monitor_test.sh <command> [args...]
|
||||
#
|
||||
# Example:
|
||||
# ./monitor_test.sh make functional-test-coverage
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
echo "Usage: $0 <command> [args...]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Start monitor
|
||||
"$SCRIPT_DIR/memory_monitor.sh" /tmp/memory_snapshot.txt &
|
||||
MONITOR_PID=$!
|
||||
trap 'kill "$MONITOR_PID" 2>/dev/null' EXIT
|
||||
|
||||
# Run command
|
||||
"$@"
|
||||
Reference in New Issue
Block a user