diff --git a/common/telemetry/config.go b/common/telemetry/config.go index ea4e31f4d8..98170bf9b4 100644 --- a/common/telemetry/config.go +++ b/common/telemetry/config.go @@ -45,14 +45,14 @@ import ( const ( // the following defaults were taken from the grpc docs as of grpc v1.46. - // they are not available programatically + // they are not available programmatically defaultReadBufferSize = 32 * 1024 defaultWriteBufferSize = 32 * 1024 defaultMinConnectTimeout = 10 * time.Second // the following defaults were taken from the otel library as of v1.7. - // they are not available programatically + // they are not available programmatically retryDefaultEnabled = true retryDefaultInitialInterval = 5 * time.Second @@ -159,6 +159,8 @@ type ( ExportConfig struct { inner exportConfig `yaml:",inline"` } + + SpanExporterType string ) // UnmarshalYAML loads the state of an ExportConfig from parsed YAML @@ -166,7 +168,7 @@ func (ec *ExportConfig) UnmarshalYAML(n *yaml.Node) error { return n.Decode(&ec.inner) } -func (ec *ExportConfig) SpanExporters() ([]otelsdktrace.SpanExporter, error) { +func (ec *ExportConfig) SpanExporters() (map[SpanExporterType]otelsdktrace.SpanExporter, error) { return ec.inner.SpanExporters() } @@ -212,10 +214,10 @@ func (g *grpcconn) dialOpts() []grpc.DialOption { } // SpanExporters builds the set of OTEL SpanExporter objects defined by the YAML -// unmarshaled into this ExportConfig object. The returned SpanExporters have +// unmarshalled into this ExportConfig object. The returned SpanExporters have // not been started. -func (ec *exportConfig) SpanExporters() ([]otelsdktrace.SpanExporter, error) { - out := make([]otelsdktrace.SpanExporter, 0, len(ec.Exporters)) +func (ec *exportConfig) SpanExporters() (map[SpanExporterType]otelsdktrace.SpanExporter, error) { + out := make(map[SpanExporterType]otelsdktrace.SpanExporter, len(ec.Exporters)) for _, expcfg := range ec.Exporters { if !strings.HasPrefix(expcfg.Kind.Signal, "trace") { continue @@ -226,7 +228,7 @@ func (ec *exportConfig) SpanExporters() ([]otelsdktrace.SpanExporter, error) { if err != nil { return nil, err } - out = append(out, spanexp) + out[SpanExporterType(expcfg.Kind.Model)] = spanexp default: return nil, fmt.Errorf("unsupported span exporter type: %T", spec) } diff --git a/common/telemetry/env.go b/common/telemetry/env.go new file mode 100644 index 0000000000..cb5f79fe3c --- /dev/null +++ b/common/telemetry/env.go @@ -0,0 +1,101 @@ +// The MIT License +// +// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. +// +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package telemetry + +import ( + "errors" + "fmt" + "strings" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + otelsdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.temporal.io/server/common/primitives" +) + +var ( + unsupportedTraceExporter = errors.New("unsupported OTEL exporter") + unsupportedTraceExporterProtocol = errors.New("unsupported OTEL exporter protocol") +) + +const ( + OtelServiceNameEnvKey = "OTEL_SERVICE_NAME" + OtelTracesExporterTypesEnvKey = "OTEL_TRACES_EXPORTER" + OtelTracesOtlpExporterType = SpanExporterType("otlp") + OtelExporterOtlpTracesProtocolEnvKey = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL" + OtelExporterOtlpTracesGrcpProtocol = "grpc" +) + +type envVarLookup = func(string) (string, bool) + +// SpanExportersFromEnv creates OTEL span exporters from environment variables. +func SpanExportersFromEnv( + envVars envVarLookup, +) (map[SpanExporterType]otelsdktrace.SpanExporter, error) { + exporters := map[SpanExporterType]otelsdktrace.SpanExporter{} + + exporterTypes, ok := envVars(OtelTracesExporterTypesEnvKey) + if !ok { + return exporters, nil + } + + for _, exporterType := range strings.Split(exporterTypes, ",") { + switch SpanExporterType(exporterType) { + case OtelTracesOtlpExporterType: + // only grpc is supported; fail if user requests a different protocol + if protocol, exists := envVars(OtelExporterOtlpTracesProtocolEnvKey); exists { + isSupported := protocol == OtelExporterOtlpTracesGrcpProtocol + if !isSupported { + return nil, fmt.Errorf("%w: %v=%v", unsupportedTraceExporterProtocol, OtelExporterOtlpTracesProtocolEnvKey, protocol) + } + } + + // other OTEL configuration env variables are picked up automatically by the exporter itself + exporters[OtelTracesOtlpExporterType] = otlptracegrpc.NewUnstarted() + default: + return nil, fmt.Errorf("%w: %v=%v", unsupportedTraceExporter, OtelTracesExporterTypesEnvKey, exporterType) + } + } + + return exporters, nil +} + +// ResourceServiceName returns the OpenTelemetry tracing service name for a Temporal service. +func ResourceServiceName( + rsn primitives.ServiceName, + envVars envVarLookup, +) string { + // map "internal-frontend" to "frontend" for the purpose of tracing + if rsn == primitives.InternalFrontendService { + rsn = primitives.FrontendService + } + + // allow custom prefix via env vars + serviceNamePrefix := "io.temporal" + if customServicePrefix, found := envVars(OtelServiceNameEnvKey); found { + serviceNamePrefix = customServicePrefix + } + + return fmt.Sprintf("%s.%s", serviceNamePrefix, string(rsn)) +} diff --git a/common/telemetry/env_test.go b/common/telemetry/env_test.go new file mode 100644 index 0000000000..add365a490 --- /dev/null +++ b/common/telemetry/env_test.go @@ -0,0 +1,124 @@ +// The MIT License +// +// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. +// +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package telemetry_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/primitives" + "go.temporal.io/server/common/telemetry" +) + +func TestSupplementTraceExportersFromEnv(t *testing.T) { + t.Run("when env variable specifies valid OTEL exporter type, add exporter", func(t *testing.T) { + exporters, err := telemetry.SpanExportersFromEnv( + func(key string) (string, bool) { + if key == telemetry.OtelTracesExporterTypesEnvKey { + return string(telemetry.OtelTracesOtlpExporterType), true + } + return "", false + }) + + require.NoError(t, err) + require.Len(t, exporters, 1) + }) + + t.Run("when env variable specifies valid OTEL exporter type but invalid protocol, return error", func(t *testing.T) { + exporters, err := telemetry.SpanExportersFromEnv( + func(key string) (string, bool) { + switch key { + case telemetry.OtelTracesExporterTypesEnvKey: + return string(telemetry.OtelTracesOtlpExporterType), true + case telemetry.OtelExporterOtlpTracesProtocolEnvKey: + return "invalid", true + } + return "", false + }) + + require.EqualError(t, err, "unsupported OTEL exporter protocol: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=invalid") + require.Empty(t, exporters) + }) + + t.Run("when env variable is specified but exporter type is not supported, return error", func(t *testing.T) { + exporters, err := telemetry.SpanExportersFromEnv( + func(key string) (string, bool) { + if key == telemetry.OtelTracesExporterTypesEnvKey { + return fmt.Sprintf("%v,%v", telemetry.OtelTracesOtlpExporterType, "nonsense"), true + } + return "", false + }) + + require.EqualError(t, err, "unsupported OTEL exporter: OTEL_TRACES_EXPORTER=nonsense") + require.Empty(t, exporters) + }) + + t.Run("when not specified, do not create any exporters", func(t *testing.T) { + exporters, err := telemetry.SpanExportersFromEnv( + func(key string) (string, bool) { + return "", false + }) + + require.NoError(t, err) + require.Empty(t, exporters) + }) +} + +func TestResourceServiceName(t *testing.T) { + t.Run("when env variable is specified, use custom service name prefix", func(t *testing.T) { + require.Equal(t, + "PREFIX.matching", + telemetry.ResourceServiceName(primitives.MatchingService, func(key string) (string, bool) { + require.Equal(t, telemetry.OtelServiceNameEnvKey, key) + return "PREFIX", true + }), + ) + }) + + t.Run("when not specified, use default prefix", func(t *testing.T) { + require.Equal(t, + "io.temporal.history", + telemetry.ResourceServiceName(primitives.HistoryService, func(key string) (string, bool) { + return "", false + }), + ) + }) + + t.Run("always use single service name for internal frontend", func(t *testing.T) { + require.Equal(t, + "PREFIX.frontend", + telemetry.ResourceServiceName(primitives.InternalFrontendService, func(key string) (string, bool) { + return "PREFIX", true + }), + ) + require.Equal(t, + "io.temporal.frontend", + telemetry.ResourceServiceName(primitives.InternalFrontendService, func(key string) (string, bool) { + return "", false + }), + ) + }) +} diff --git a/develop/docs/tracing.md b/develop/docs/tracing.md index 80a2679ff9..022b88afa6 100644 --- a/develop/docs/tracing.md +++ b/develop/docs/tracing.md @@ -16,18 +16,21 @@ itself](https://github.com/open-telemetry/opentelemetry-specification/blob/main/ ## Configuring No trace exporters are configured by default and thus trace data is neither -collected nor emitted without additional configuration added to the server's -yaml configuration files. +collected nor emitted without additional configuration. -The server now supports a new `otel` YAML stanza which is used to configure a -set of process-wide exporters. In OpenTelemetry, the concept of an "exporter" is +In OpenTelemetry, the concept of an "exporter" is abstract. The concrete implementation of an exporter is determined by a -3-tuple of values: the exporter signal, model, and protocol. In OTEL, a "signal" -is one of traces, metrics, or logs (in this document we will only deal with -traces), "model" indicates the abstract data model for the span and trace data -being exported, and the "protocol" specifies the concrete application protocol -binding for the indicated model. Temporal is known to support exporting trace -data as defined by otlp over either grpc or http. +3-tuple of values: the exporter signal, model, and protocol: +- a "signal" is one of traces, metrics, or logs (in this document we will only deal with traces), +- "model" indicates the abstract data model for the span and trace data being exported, +- and the "protocol" specifies the concrete application protocol binding for the indicated model. + +Temporal is known to support exporting trace data as defined by otlp over grpc. + +### Configuration File + +The server supports an `otel` YAML stanza which is used to configure a +set of process-wide exporters. A common configuration is to emit tracing data to an agent such as the [otel-collector](https://opentelemetry.io/docs/collector/) running locally. To @@ -70,12 +73,34 @@ fields can be found in [config_test.go](../../common/telemetry/config_test.go) and are mostly related to the underlying gRPC client configuration (retries, timeouts, etc). -Note that the Go OTEL SDK will also read a well-known set of environment -variables for configuration. So if you prefer setting environment variables to -writing YAML then you can use the [variables defined in the OTEL -spec](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md). -If environment variables conflict with YAML-provided configuration then the YAML -takes precedence. +### Environment Variables + +#### Creating Exporter + +An OTEL span exporter can also be configured via environment variables: [OTEL_TRACES_EXPORTER]( +https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#exporter-selection) +creates a span exporter. + +``` +OTEL_TRACES_EXPORTER=otlp +``` + +Note that if the configuration file already defines a traces exporter, no additional exporter +will be created. + +#### Configuring Exporter + +The Go OTEL SDK will also read a well-known set of environment variables for the configuration +of the exporter. So if you prefer setting environment variables to writing YAML then you can use the +[variables defined in the OTEL spec](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/). + +For example: +``` +OTEL_SERVICE_NAME=my-service OTEL_EXPORTER_OTLP_TRACES_INSECURE=true +``` + +**NOTE: If an environment variable conflicts with YAML-provided configuration then the environment +variable takes precedence.** ## Instrumenting diff --git a/temporal/fx.go b/temporal/fx.go index 6c6387d1af..f2ec10295f 100644 --- a/temporal/fx.go +++ b/temporal/fx.go @@ -28,7 +28,7 @@ import ( "context" "errors" "fmt" - "strings" + "os" "github.com/pborman/uuid" "go.opentelemetry.io/otel" @@ -902,10 +902,20 @@ var TraceExportModule = fx.Options( }), fx.Provide(func(lc fx.Lifecycle, c *config.Config) ([]otelsdktrace.SpanExporter, error) { - exporters, err := c.ExporterConfig.SpanExporters() + exportersByType, err := c.ExporterConfig.SpanExporters() if err != nil { return nil, err } + + exportersByTypeFromEnv, err := telemetry.SpanExportersFromEnv(os.LookupEnv) + if err != nil { + return nil, err + } + + // config-defined exporters override env-defined exporters with the same type + maps.Copy(exportersByType, exportersByTypeFromEnv) + + exporters := maps.Values(exportersByType) lc.Append(fx.Hook{ OnStart: startAll(exporters), OnStop: shutdownAll(exporters), @@ -948,21 +958,14 @@ var ServiceTracingModule = fx.Options( fx.Provide( fx.Annotate( func(rsn primitives.ServiceName, rsi resource.InstanceID) (*otelresource.Resource, error) { - // map "internal-frontend" to "frontend" for the purpose of tracing - if rsn == primitives.InternalFrontendService { - rsn = primitives.FrontendService - } - serviceName := string(rsn) - if !strings.HasPrefix(serviceName, "io.temporal.") { - serviceName = fmt.Sprintf("io.temporal.%s", serviceName) - } attrs := []attribute.KeyValue{ - semconv.ServiceNameKey.String(serviceName), + semconv.ServiceNameKey.String(telemetry.ResourceServiceName(rsn, os.LookupEnv)), semconv.ServiceVersionKey.String(headers.ServerVersion), } if rsi != "" { attrs = append(attrs, semconv.ServiceInstanceIDKey.String(string(rsi))) } + return otelresource.New(context.Background(), otelresource.WithProcess(), otelresource.WithOS(),