Grafana Tempo (#7006)

## What changed?
<!-- Describe what has changed in this PR -->

Added Grafana Tempo - an OTEL collector - to the developer setup.

Since we're already using Grafana, Tempo seemed like a logical choice.

## Why?
<!-- Tell your future self why have you made these changes -->

It allows developers to see all the RPC calls they are making (e.g. when
running the server locally or a functional test).

Tempo allows filtering by workflow ID. _(note that not _all_ places are
tagged yet, but lots of them)_

## How did you test it?
<!-- How have you verified this change? Tested locally? Added a unit
test? Checked in staging env? -->

```
make start-grafana-tempo
make OTEL=true start
```
```
tctl --ns default namespace register
tctl wf start --tq test --wid test -wt test
```

<img width="3352" alt="image"
src="https://github.com/user-attachments/assets/5809691a-da32-4a04-8e80-700f8acd47e5"
/>

And running a functional test from the IDE:

<img width="3346" alt="image"
src="https://github.com/user-attachments/assets/061bfaf3-0a34-4b4e-a06c-497dfa3a948a"
/>


## Potential risks
<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->

## Documentation
<!-- Have you made sure this change doesn't falsify anything currently
stated in `docs/`? If significant
new behavior is added, have you described that in `docs/`? -->

Updated docs/development/tracing.md

## Is hotfix candidate?
<!-- Is this PR a hotfix candidate or does it require a notification to
be sent to the broader community? (Yes/No) -->

Makefile

@@ -74,6 +74,14 @@ STAMPDIR := .stamp
 export PATH := $(ROOT)/$(LOCALBIN):$(PATH)
 GOINSTALL := GOBIN=$(ROOT)/$(LOCALBIN) go install
 
+OTEL ?= false
+ifeq ($(OTEL),true)
+	export OTEL_BSP_SCHEDULE_DELAY=0
+	export OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
+	export OTEL_TRACES_EXPORTER=otlp
+	export TEMPORAL_OTEL_DEBUG=true
+endif
+
 MODULE_ROOT := $(lastword $(shell grep -e "^module " go.mod))
 COLOR := "\e[1;36m%s\e[0m\n"
 RED :=   "\e[1;31m%s\e[0m\n"

common/telemetry/env.go

@@ -73,6 +73,8 @@ func SpanExportersFromEnv(
 
 			// other OTEL configuration env variables are picked up automatically by the exporter itself
 			exporters[OtelTracesOtlpExporterType] = otlptracegrpc.NewUnstarted()
+		case "none":
+			// ignored
 		default:
 			return nil, fmt.Errorf("%w: %v=%v", unsupportedTraceExporter, OtelTracesExporterTypesEnvKey, exporterType)
 		}

develop/docker-compose/docker-compose.darwin.yml

@@ -12,6 +12,14 @@ services:
       - "3000:3000"
     networks:
       - temporal-dev-network
+  tempo:
+    depends_on:
+      - grafana
+    ports:
+      - "3200:3200" # tempo
+      - "4317:4317" # otlp grpc
+    networks:
+      - temporal-dev-network
   temporal-ui:
     environment:
       - TEMPORAL_ADDRESS=host.docker.internal:7233

develop/docker-compose/docker-compose.jaeger.yml

@@ -1,11 +0,0 @@
-# Include this file to run Jaeger as one more dependency:
-# docker-compose -f docker-compose.yml -f docker-compose.linux.yml -f docker-compose.jaeger.yml up
-services:
-  jaeger:
-    image: jaegertracing/all-in-one:latest
-    container_name: temporal-dev-jaeger
-    ports:
-      - "6831:6831/udp"
-      - "16686:16686"
-    networks:
-      - temporal-dev-network

develop/docker-compose/docker-compose.linux.yml

@@ -6,6 +6,10 @@ services:
       - ./prometheus-linux:/etc/prometheus
   grafana:
     network_mode: host
+  tempo:
+    depends_on:
+      - grafana
+    network_mode: host
   temporal-ui:
     environment:
       - TEMPORAL_ADDRESS=localhost:7233

develop/docker-compose/docker-compose.windows.yml

@@ -12,6 +12,14 @@ services:
       - "3000:3000"
     networks:
       - temporal-dev-network
+  tempo:
+    depends_on:
+      - grafana
+    ports:
+      - "3200:3200" # tempo
+      - "4317:4317" # otlp grpc
+    networks:
+      - temporal-dev-network
   temporal-ui:
     environment:
       - TEMPORAL_ADDRESS=host.docker.internal:7233

develop/docker-compose/docker-compose.yml

@@ -75,6 +75,13 @@ services:
       - prometheus
     volumes:
       - ./grafana/provisioning/:/etc/grafana/provisioning/
+  tempo:
+    image: grafana/tempo:latest
+    container_name: temporal-dev-grafana-tempo
+    command:
+      - "-config.file=/etc/tempo.yaml"
+    volumes:
+      - ./grafana/provisioning/tempo/tempo.yaml:/etc/tempo.yaml
   temporal-ui:
     image: temporalio/ui:latest
     container_name: temporal-dev-ui

develop/docker-compose/grafana/provisioning/datasources/prometheus.yml

@@ -9,4 +9,14 @@ datasources:
   type: prometheus
   url: http://prometheus:9090
   access: proxy
+  isDefault: false
+- name: Tempo-Linux
+  type: tempo
+  access: proxy
+  url: http://localhost:3200
+  isDefault: false
+- name: Tempo-Darwin
+  type: tempo
+  access: proxy
+  url: http://tempo:3200
   isDefault: false

develop/docker-compose/grafana/provisioning/tempo/tempo.yaml

@@ -0,0 +1,83 @@
+# Copied from https://github.com/grafana/tempo/blob/main/example/docker-compose/local/tempo.yaml
+
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+cache:
+  background:
+    writeback_goroutines: 5
+  caches:
+    - roles:
+        - frontend-search
+      memcached:
+        host: memcached:11211
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+      duration_slo: 5s
+      throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 100ms
+  metrics:
+    max_duration: 120h                # maximum duration of a metrics query, increase for local setups
+    query_backend_after: 5m
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 24h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+  processor:
+    local_blocks:
+      filter_server_spans: false
+      flush_to_storage: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /var/tempo/wal             # where to store the wal locally
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
+      generate_native_histograms: both

develop/docker-compose/prometheus-darwin/prometheus.yml

@@ -10,3 +10,7 @@ scrape_configs:
       - targets:
           # port should match value from server config listenAddress.
           - host.docker.internal:8000
+  - job_name: tempo
+    static_configs:
+      - targets:
+          - host.docker.internal:3200

develop/docker-compose/prometheus-linux/prometheus.yml

@@ -10,3 +10,7 @@ scrape_configs:
       - targets:
           # port should match value from server config listenAddress.
           - localhost:8000
+  - job_name: tempo
+    static_configs:
+      - targets:
+          - localhost:3200

docs/development/tracing.md

@@ -2,9 +2,8 @@
 
 The Temporal server supports ability to configure OTEL trace exporters to
 support emitting spans and traces for observability. More specifically, the
-server uses the [Go Open Telemetry
-library](https://github.com/open-telemetry/opentelemetry-go) for instrumentation
-and multi-protocol multi-model telemetry exporting. This document is intended to
+server uses the [Go Open Telemetry library](https://github.com/open-telemetry/opentelemetry-go)
+for instrumentation and multi-protocol multi-model telemetry exporting. This document is intended to
 help developers understand how to configure exporters and instrument their code.
 A full exploration of tracing and telemetry is out of scope of this document and
 the reader is referred to [external reference
@@ -15,16 +14,12 @@ itself](https://github.com/open-telemetry/opentelemetry-specification/blob/main/
 
 ## Quickstart
 
-1. Run [`jaeger-all-in-one`](https://www.jaegertracing.io/download/) (either from a binary, or using the docker image).
-2. View the Jaeger UI at http://localhost:16686/search
-3. Set these environment variables:
-    ```
-    export OTEL_TRACES_EXPORTER=otlp
-    export OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
-    export TEMPORAL_OTEL_DEBUG=true
-    ```
-4. Start the server using `make start`
-5. Refresh the Jaeger UI. You should see traces from Temporal services.
+1. Run `make start-dependencies` (which starts Grafana Tempo)
+2. Start the server using `make OTEL=true start` (or any other start-x command)
+3. Visit http://localhost:3000/explore and select "Tempo" from the datasource dropdown.
+
+tip: use [TraceQL](https://grafana.com/docs/tempo/latest/traceql/)
+`{ .temporalWorkflowID =~ "<WF-ID>.*" }` to find the traces for your workflow 
 
 ## Configuring
 

temporal/fx.go

@@ -859,6 +859,12 @@ func verifyPersistenceCompatibleVersion(config config.Persistence, persistenceSe
 	return nil
 }
 
+type SpanExporterInputs struct {
+	fx.In
+	Lifecycyle fx.Lifecycle
+	Config     *config.Config `optional:"true"`
+}
+
 // TraceExportModule holds process-global telemetry fx state defining the set of
 // OTEL trace/span exporters used by tracing instrumentation. The following
 // types can be overriden/augmented with fx.Replace/fx.Decorate:
@@ -873,10 +879,14 @@ var TraceExportModule = fx.Options(
 		)
 	}),
 
-	fx.Provide(func(lc fx.Lifecycle, c *config.Config) ([]otelsdktrace.SpanExporter, error) {
-		exportersByType, err := c.ExporterConfig.SpanExporters()
-		if err != nil {
-			return nil, err
+	fx.Provide(func(inputs SpanExporterInputs) ([]otelsdktrace.SpanExporter, error) {
+		exportersByType := map[telemetry.SpanExporterType]otelsdktrace.SpanExporter{}
+		if inputs.Config != nil {
+			var err error
+			exportersByType, err = inputs.Config.ExporterConfig.SpanExporters()
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		exportersByTypeFromEnv, err := telemetry.SpanExportersFromEnv(os.LookupEnv)
@@ -888,7 +898,7 @@ var TraceExportModule = fx.Options(
 		maps.Copy(exportersByType, exportersByTypeFromEnv)
 
 		exporters := expmaps.Values(exportersByType)
-		lc.Append(fx.Hook{
+		inputs.Lifecycyle.Append(fx.Hook{
 			OnStart: startAll(exporters),
 			OnStop:  shutdownAll(exporters),
 		})

tests/testcore/onebox.go

@@ -38,7 +38,6 @@ import (
 	"testing"
 	"time"
 
-	otelsdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"go.temporal.io/api/operatorservice/v1"
 	"go.temporal.io/api/workflowservice/v1"
 	"go.temporal.io/server/api/adminservice/v1"
@@ -120,7 +119,6 @@ type (
 		esClient                         esclient.Client
 		mockAdminClient                  map[string]adminservice.AdminServiceClient
 		namespaceReplicationTaskExecutor namespace.ReplicationTaskExecutor
-		spanExporters                    []otelsdktrace.SpanExporter
 		tlsConfigProvider                *encryption.FixedTLSConfigProvider
 		captureMetricsHandler            *metricstest.CaptureHandler
 		hostsByProtocolByService         map[transferProtocol]map[primitives.ServiceName]static.Hosts
@@ -188,7 +186,6 @@ type (
 		ESClient                         esclient.Client
 		MockAdminClient                  map[string]adminservice.AdminServiceClient
 		NamespaceReplicationTaskExecutor namespace.ReplicationTaskExecutor
-		SpanExporters                    []otelsdktrace.SpanExporter
 		DynamicConfigOverrides           map[dynamicconfig.Key]interface{}
 		TLSConfigProvider                *encryption.FixedTLSConfigProvider
 		CaptureMetricsHandler            *metricstest.CaptureHandler
@@ -254,7 +251,6 @@ func newTemporal(t *testing.T, params *TemporalParams) *TemporalImpl {
 		workerConfig:                     params.WorkerConfig,
 		mockAdminClient:                  params.MockAdminClient,
 		namespaceReplicationTaskExecutor: params.NamespaceReplicationTaskExecutor,
-		spanExporters:                    params.SpanExporters,
 		tlsConfigProvider:                params.TLSConfigProvider,
 		captureMetricsHandler:            params.CaptureMetricsHandler,
 		dcClient:                         dynamicconfig.NewMemoryClient(),
@@ -415,7 +411,7 @@ func (c *TemporalImpl) startFrontend() {
 			fx.Provide(func() esclient.Client { return c.esClient }),
 			fx.Provide(c.GetTLSConfigProvider),
 			fx.Provide(c.GetTaskCategoryRegistry),
-			fx.Supply(c.spanExporters),
+			temporal.TraceExportModule,
 			temporal.ServiceTracingModule,
 			frontend.Module,
 			fx.Populate(&namespaceRegistry, &rpcFactory, &historyRawClient, &matchingRawClient, &grpcResolver),
@@ -486,9 +482,8 @@ func (c *TemporalImpl) startHistory() {
 			fx.Provide(func() esclient.Client { return c.esClient }),
 			fx.Provide(c.GetTLSConfigProvider),
 			fx.Provide(c.GetTaskCategoryRegistry),
-			fx.Supply(c.spanExporters),
+			temporal.TraceExportModule,
 			temporal.ServiceTracingModule,
-
 			history.QueueModule,
 			history.Module,
 			replication.Module,
@@ -539,7 +534,7 @@ func (c *TemporalImpl) startMatching() {
 			fx.Provide(c.GetTLSConfigProvider),
 			fx.Provide(resource.DefaultSnTaggedLoggerProvider),
 			fx.Provide(c.GetTaskCategoryRegistry),
-			fx.Supply(c.spanExporters),
+			temporal.TraceExportModule,
 			temporal.ServiceTracingModule,
 			matching.Module,
 			temporal.FxLogAdapter,
@@ -602,8 +597,6 @@ func (c *TemporalImpl) startWorker() {
 			fx.Provide(func() *esclient.Config { return c.esConfig }),
 			fx.Provide(c.GetTLSConfigProvider),
 			fx.Provide(c.GetTaskCategoryRegistry),
-			fx.Supply(c.spanExporters),
-			temporal.ServiceTracingModule,
 			worker.Module,
 			temporal.FxLogAdapter,
 			c.getFxOptionsForService(primitives.WorkerService),