mirror of
https://github.com/temporalio/temporal.git
synced 2026-08-30 18:41:49 -07:00
1827 lines
72 KiB
Go
1827 lines
72 KiB
Go
package workerdeployment
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
|
|
"github.com/google/uuid"
|
|
computepb "go.temporal.io/api/compute/v1"
|
|
deploymentpb "go.temporal.io/api/deployment/v1"
|
|
enumspb "go.temporal.io/api/enums/v1"
|
|
"go.temporal.io/api/serviceerror"
|
|
sdkclient "go.temporal.io/sdk/client"
|
|
sdklog "go.temporal.io/sdk/log"
|
|
"go.temporal.io/sdk/temporal"
|
|
"go.temporal.io/sdk/workflow"
|
|
deploymentspb "go.temporal.io/server/api/deployment/v1"
|
|
"go.temporal.io/server/common/metrics"
|
|
"go.temporal.io/server/common/worker_versioning"
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
)
|
|
|
|
const (
|
|
// The actual limit is set in dynamic configs, this is only used in case we cannot read the DC.
|
|
defaultMaxVersions = 100
|
|
)
|
|
|
|
type (
|
|
// SignalHandler encapsulates the signal handling logic
|
|
SignalHandler struct {
|
|
signalSelector workflow.Selector
|
|
processingSignals int
|
|
}
|
|
|
|
// WorkflowRunner holds the local state while running a deployment-series workflow
|
|
WorkflowRunner struct {
|
|
*deploymentspb.WorkerDeploymentWorkflowArgs
|
|
a *Activities
|
|
logger sdklog.Logger
|
|
metrics sdkclient.MetricsHandler
|
|
lock workflow.Mutex
|
|
deleteDeployment bool
|
|
unsafeMaxVersion func() int
|
|
// stateChanged is used to track if the state of the workflow has undergone a local state change since the last signal/update.
|
|
// This prevents a workflow from continuing-as-new if the state has not changed.
|
|
stateChanged bool
|
|
signalHandler *SignalHandler
|
|
forceCAN bool
|
|
// Optional override state for force-CaN
|
|
overrideState *deploymentspb.WorkerDeploymentLocalState
|
|
// workflowVersion is set at workflow start based on the dynamic config of the worker
|
|
// that completes the first task. It remains constant for the lifetime of the run and
|
|
// only updates when the workflow performs continue-as-new.
|
|
workflowVersion DeploymentWorkflowVersion
|
|
}
|
|
)
|
|
|
|
// This workflow is implemented in a way such that it always CaNs after some
|
|
// history events are added to it and when it has no pending work to do. This is to keep the
|
|
// history clean so that we have less concern about backwards and forwards compatibility.
|
|
// In steady state (i.e. absence of ongoing updates or signals) the wf should only have
|
|
// a single wft in the history.
|
|
func Workflow(ctx workflow.Context, unsafeWorkflowVersionGetter func() DeploymentWorkflowVersion, unsafeMaxVersion func() int, args *deploymentspb.WorkerDeploymentWorkflowArgs) error {
|
|
workflowRunner := &WorkflowRunner{
|
|
WorkerDeploymentWorkflowArgs: args,
|
|
workflowVersion: getWorkflowVersion(ctx, unsafeWorkflowVersionGetter),
|
|
a: nil,
|
|
logger: sdklog.With(workflow.GetLogger(ctx), "wf-namespace", args.NamespaceName),
|
|
metrics: workflow.GetMetricsHandler(ctx).WithTags(map[string]string{"namespace": args.NamespaceName}),
|
|
lock: workflow.NewMutex(ctx),
|
|
unsafeMaxVersion: unsafeMaxVersion,
|
|
signalHandler: &SignalHandler{
|
|
signalSelector: workflow.NewSelector(ctx),
|
|
},
|
|
}
|
|
|
|
return workflowRunner.run(ctx)
|
|
}
|
|
|
|
func getWorkflowVersion(ctx workflow.Context, unsafeWorkflowVersionGetter func() DeploymentWorkflowVersion) DeploymentWorkflowVersion {
|
|
if workflow.GetVersion(ctx, "workflowVersionAdded", workflow.DefaultVersion, 0) >= 0 {
|
|
var ver DeploymentWorkflowVersion
|
|
err := workflow.MutableSideEffect(ctx, "workflowVersion",
|
|
func(_ workflow.Context) any { return unsafeWorkflowVersionGetter() },
|
|
func(a, b any) bool { return a == b }).
|
|
Get(&ver)
|
|
if err == nil {
|
|
return ver
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (d *WorkflowRunner) hasMinVersion(version DeploymentWorkflowVersion) bool {
|
|
return d.workflowVersion >= version
|
|
}
|
|
|
|
func (d *WorkflowRunner) listenToSignals(ctx workflow.Context) {
|
|
forceCANSignalChannel := workflow.GetSignalChannel(ctx, ForceCANSignalName)
|
|
syncVersionSummaryChannel := workflow.GetSignalChannel(ctx, SyncVersionSummarySignal)
|
|
propagationCompleteChannel := workflow.GetSignalChannel(ctx, PropagationCompleteSignal)
|
|
|
|
d.signalHandler.signalSelector.AddReceive(forceCANSignalChannel, func(c workflow.ReceiveChannel, more bool) {
|
|
d.signalHandler.processingSignals++
|
|
defer func() { d.signalHandler.processingSignals-- }()
|
|
|
|
var args *deploymentspb.ForceCANDeploymentSignalArgs
|
|
c.Receive(ctx, &args)
|
|
d.forceCAN = true
|
|
|
|
// Apply override state if provided
|
|
if args.GetOverrideState() != nil {
|
|
d.overrideState = args.GetOverrideState()
|
|
}
|
|
})
|
|
d.signalHandler.signalSelector.AddReceive(syncVersionSummaryChannel, func(c workflow.ReceiveChannel, more bool) {
|
|
d.signalHandler.processingSignals++
|
|
defer func() { d.signalHandler.processingSignals-- }()
|
|
var summary *deploymentspb.WorkerDeploymentVersionSummary
|
|
c.Receive(ctx, &summary)
|
|
d.syncVersionSummaryFromVersionWorkflow(ctx, summary)
|
|
d.setStateChanged()
|
|
})
|
|
d.signalHandler.signalSelector.AddReceive(propagationCompleteChannel, func(c workflow.ReceiveChannel, more bool) {
|
|
d.signalHandler.processingSignals++
|
|
defer func() { d.signalHandler.processingSignals-- }()
|
|
var completion *deploymentspb.PropagationCompletionInfo
|
|
c.Receive(ctx, &completion)
|
|
d.handlePropagationComplete(completion)
|
|
d.setStateChanged()
|
|
})
|
|
|
|
// Keep waiting for signals, when it's time to CaN the main goroutine will exit.
|
|
for {
|
|
d.signalHandler.signalSelector.Select(ctx)
|
|
}
|
|
}
|
|
|
|
// syncVersionSummary ensures the version summary in the deployment workflow stays consistent
|
|
// with the version workflow. This helps prevent discrepancies if they ever fall out of sync.
|
|
func (d *WorkflowRunner) syncVersionSummaryFromVersionWorkflow(ctx workflow.Context, summary *deploymentspb.WorkerDeploymentVersionSummary) {
|
|
existing, ok := d.State.Versions[summary.GetVersion()]
|
|
if !ok {
|
|
d.logger.Error("received summary for a non-existing version, ignoring it", "version", summary.GetVersion())
|
|
return
|
|
}
|
|
|
|
// Preserve create_request_id since the version workflow doesn't know about it.
|
|
summary.CreateRequestId = existing.GetCreateRequestId()
|
|
d.State.Versions[summary.GetVersion()] = summary
|
|
if workflow.GetVersion(ctx, "update-memo-with-summary", workflow.DefaultVersion, 0) != workflow.DefaultVersion {
|
|
if err := d.updateMemo(ctx); err != nil {
|
|
d.logger.Error("failed to update memo", "error", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// handlePropagationComplete handles the propagation complete signal from version workflows
|
|
func (d *WorkflowRunner) handlePropagationComplete(completion *deploymentspb.PropagationCompletionInfo) {
|
|
buildID := completion.BuildId
|
|
revisionNumber := completion.RevisionNumber
|
|
|
|
if d.State.PropagatingRevisions == nil {
|
|
d.logger.Error("Received propagation complete signal, but no in-progress propagations found",
|
|
"revision", revisionNumber,
|
|
"build_id", buildID)
|
|
return
|
|
}
|
|
|
|
// Remove this revision from in-progress tracking for this build
|
|
if revisions, ok := d.State.PropagatingRevisions[buildID]; ok {
|
|
found := false
|
|
// Find and remove the revision number
|
|
filteredRevisions := make([]int64, 0, len(revisions.RevisionNumbers))
|
|
for _, rev := range revisions.RevisionNumbers {
|
|
if rev != revisionNumber {
|
|
filteredRevisions = append(filteredRevisions, rev)
|
|
} else {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
d.logger.Error("Received propagation complete signal, but this propagation is not in-progress",
|
|
"revision", revisionNumber,
|
|
"build_id", buildID)
|
|
}
|
|
|
|
if len(filteredRevisions) == 0 {
|
|
// Clean up empty build id entries
|
|
delete(d.State.PropagatingRevisions, buildID)
|
|
} else {
|
|
revisions.RevisionNumbers = filteredRevisions
|
|
}
|
|
} else {
|
|
d.logger.Error("Received propagation complete signal, but no in-progress propagations found for this version",
|
|
"revision", revisionNumber,
|
|
"build_id", buildID)
|
|
}
|
|
|
|
d.logger.Info("Propagation completed for revision",
|
|
"revision", revisionNumber,
|
|
"build_id", buildID)
|
|
}
|
|
|
|
func (d *WorkflowRunner) updateVersionSummary(summary *deploymentspb.WorkerDeploymentVersionSummary) {
|
|
if _, ok := d.State.Versions[summary.GetVersion()]; !ok {
|
|
d.logger.Error("received summary for a non-existing version, ignoring it", "version", summary.GetVersion())
|
|
return
|
|
}
|
|
|
|
// Preserve create_time and first_activation_time if they exist in current summary. This is to ensure that if the version
|
|
// had already been activated before, we don't override the first activation time by setting it to a wrong value.
|
|
if existingSummary := d.State.Versions[summary.GetVersion()]; existingSummary.GetCreateTime() != nil {
|
|
summary.CreateTime = existingSummary.GetCreateTime()
|
|
|
|
if existingSummary.GetFirstActivationTime() != nil {
|
|
summary.FirstActivationTime = existingSummary.GetFirstActivationTime()
|
|
}
|
|
}
|
|
|
|
d.State.Versions[summary.GetVersion()] = summary
|
|
}
|
|
|
|
func (d *WorkflowRunner) run(ctx workflow.Context) error {
|
|
// TODO(carlydf): remove verbose logging
|
|
d.logger.Info("Raw workflow state at start",
|
|
"state_nil", d.State == nil,
|
|
"create_time_nil", d.GetState().GetCreateTime() == nil,
|
|
"routing_config_nil", d.GetState().GetRoutingConfig() == nil,
|
|
"raw_state", d.State,
|
|
"workflow_id", workflow.GetInfo(ctx).WorkflowExecution.ID,
|
|
"run_id", workflow.GetInfo(ctx).WorkflowExecution.RunID)
|
|
|
|
if d.GetState().GetCreateTime() == nil ||
|
|
d.GetState().GetRoutingConfig() == nil ||
|
|
d.GetState().GetConflictToken() == nil {
|
|
if d.State == nil {
|
|
d.State = &deploymentspb.WorkerDeploymentLocalState{}
|
|
}
|
|
if d.State.CreateTime == nil {
|
|
d.State.CreateTime = timestamppb.New(workflow.Now(ctx))
|
|
}
|
|
if d.State.RoutingConfig == nil {
|
|
d.State.RoutingConfig = &deploymentpb.RoutingConfig{CurrentVersion: worker_versioning.UnversionedVersionId}
|
|
}
|
|
if d.State.ConflictToken == nil {
|
|
d.State.ConflictToken, _ = workflow.Now(ctx).MarshalBinary()
|
|
}
|
|
|
|
// updating the memo since the RoutingConfig is updated
|
|
if err := d.updateMemo(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
d.metrics.Counter(metrics.WorkerDeploymentCreated.Name()).Inc(1)
|
|
}
|
|
if d.State.Versions == nil {
|
|
d.State.Versions = make(map[string]*deploymentspb.WorkerDeploymentVersionSummary)
|
|
}
|
|
|
|
// TODO(carlydf): remove verbose logging
|
|
d.logger.Info("Starting workflow run",
|
|
"create_time", d.State.GetCreateTime(),
|
|
"routing_config", d.State.GetRoutingConfig(),
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
"current_version", d.State.GetRoutingConfig().GetCurrentVersion(),
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
"ramping_version", d.State.GetRoutingConfig().GetRampingVersion())
|
|
|
|
err := workflow.SetQueryHandler(ctx, QueryDescribeDeployment, func() (*deploymentspb.QueryDescribeWorkerDeploymentResponse, error) {
|
|
if d.deleteDeployment {
|
|
return nil, errors.New(errDeploymentDeleted)
|
|
}
|
|
return &deploymentspb.QueryDescribeWorkerDeploymentResponse{
|
|
State: d.State,
|
|
}, nil
|
|
})
|
|
if err != nil {
|
|
d.logger.Info("SetQueryHandler failed for WorkerDeployment workflow with error: " + err.Error())
|
|
return err
|
|
}
|
|
|
|
err = workflow.SetQueryHandler(ctx, QueryCreateRequestID, func() (*deploymentspb.CreateRequestIDQueryResponse, error) {
|
|
if d.deleteDeployment {
|
|
return nil, errors.New(errDeploymentDeleted)
|
|
}
|
|
return &deploymentspb.CreateRequestIDQueryResponse{
|
|
RequestId: d.GetState().GetCreateRequestId(),
|
|
ConflictToken: d.State.GetConflictToken(),
|
|
}, nil
|
|
})
|
|
if err != nil {
|
|
d.logger.Info("SetQueryHandler failed for WorkerDeployment create request-id query with error: " + err.Error())
|
|
return err
|
|
}
|
|
|
|
if err := workflow.SetUpdateHandlerWithOptions(
|
|
ctx,
|
|
CreateWorkerDeployment,
|
|
d.handleCreateWorkerDeployment,
|
|
workflow.UpdateHandlerOptions{
|
|
Validator: d.validateCreateWorkerDeployment,
|
|
},
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := workflow.SetUpdateHandlerWithOptions(
|
|
ctx,
|
|
CreateWorkerDeploymentVersion,
|
|
d.handleCreateWorkerDeploymentVersion,
|
|
workflow.UpdateHandlerOptions{
|
|
Validator: d.validateCreateWorkerDeploymentVersion,
|
|
},
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := workflow.SetUpdateHandler(
|
|
ctx,
|
|
RegisterWorkerInWorkerDeployment,
|
|
d.handleRegisterWorker,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := workflow.SetUpdateHandlerWithOptions(
|
|
ctx,
|
|
SetCurrentVersion,
|
|
d.handleSetCurrent,
|
|
workflow.UpdateHandlerOptions{
|
|
Validator: d.validateSetCurrent,
|
|
},
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := workflow.SetUpdateHandlerWithOptions(
|
|
ctx,
|
|
SetRampingVersion,
|
|
d.handleSetRampingVersion,
|
|
workflow.UpdateHandlerOptions{
|
|
Validator: d.validateSetRampingVersion,
|
|
},
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := workflow.SetUpdateHandlerWithOptions(
|
|
ctx,
|
|
SetManagerIdentity,
|
|
d.handleSetManager,
|
|
workflow.UpdateHandlerOptions{
|
|
Validator: d.validateSetManager,
|
|
},
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := workflow.SetUpdateHandlerWithOptions(
|
|
ctx,
|
|
DeleteVersion,
|
|
d.handleDeleteVersion,
|
|
workflow.UpdateHandlerOptions{
|
|
Validator: d.validateDeleteVersion,
|
|
},
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := workflow.SetUpdateHandlerWithOptions(
|
|
ctx,
|
|
DeleteDeployment,
|
|
d.handleDeleteDeployment,
|
|
workflow.UpdateHandlerOptions{
|
|
Validator: d.validateDeleteDeployment,
|
|
},
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Listen to signals in a different goroutine to make business logic clearer
|
|
workflow.Go(ctx, d.listenToSignals)
|
|
|
|
// Wait until we can continue as new or are cancelled. The workflow will continue-as-new iff
|
|
// there are no pending updates/signals and the state has changed.
|
|
err = workflow.Await(ctx, func() bool {
|
|
canContinue := d.deleteDeployment || // deployment is deleted -> it's ok to drop all signals and updates.
|
|
// There is no pending signal or update, but the state is dirty or forceCaN is requested:
|
|
(!d.signalHandler.signalSelector.HasPending() && d.signalHandler.processingSignals == 0 && workflow.AllHandlersFinished(ctx) &&
|
|
(d.forceCAN || d.stateChanged || workflow.GetInfo(ctx).GetContinueAsNewSuggested()))
|
|
|
|
// TODO(carlydf): remove verbose logging
|
|
if canContinue {
|
|
d.logger.Info("Workflow can continue as new",
|
|
"workflow_id", workflow.GetInfo(ctx).WorkflowExecution.ID,
|
|
"run_id", workflow.GetInfo(ctx).WorkflowExecution.RunID,
|
|
"delete_deployment", d.deleteDeployment,
|
|
"has_pending_signals", d.signalHandler.signalSelector.HasPending(),
|
|
"processing_signals", d.signalHandler.processingSignals,
|
|
"all_handlers_finished", workflow.AllHandlersFinished(ctx),
|
|
"force_can", d.forceCAN,
|
|
"state_changed", d.stateChanged,
|
|
"routing_config", d.State.GetRoutingConfig())
|
|
}
|
|
return canContinue
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if d.deleteDeployment {
|
|
return nil
|
|
}
|
|
|
|
// TODO(carlydf): remove verbose logging
|
|
d.logger.Info("Continuing workflow as new",
|
|
"create_time", d.State.GetCreateTime(),
|
|
"routing_config", d.State.GetRoutingConfig(),
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
"current_version", d.State.GetRoutingConfig().GetCurrentVersion(),
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
"ramping_version", d.State.GetRoutingConfig().GetRampingVersion(),
|
|
"state_changed", d.stateChanged,
|
|
"force_can", d.forceCAN,
|
|
"workflow_id", workflow.GetInfo(ctx).WorkflowExecution.ID,
|
|
"run_id", workflow.GetInfo(ctx).WorkflowExecution.RunID)
|
|
|
|
// We perform a continue-as-new after each update and signal is handled to ensure compatibility
|
|
// even if the server rolls back to a previous minor version. By continuing-as-new,
|
|
// we pass the current state as input to the next workflow execution, resulting in a new
|
|
// workflow history with just two initial events. This minimizes the risk of NDE (Non-Deterministic Execution)
|
|
// errors during server rollbacks.
|
|
|
|
// Apply override state if provided during force-CaN
|
|
if d.overrideState != nil {
|
|
d.State = d.overrideState
|
|
}
|
|
return workflow.NewContinueAsNewError(ctx, WorkerDeploymentWorkflowType, d.WorkerDeploymentWorkflowArgs)
|
|
}
|
|
|
|
func (d *WorkflowRunner) preUpdateChecks(ctx workflow.Context) error {
|
|
err := d.ensureNotDeleted()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if workflow.GetInfo(ctx).GetContinueAsNewSuggested() {
|
|
// History is too large, do not accept new updates until wf CaNs.
|
|
// Since this needs workflow context we cannot do it in validators.
|
|
return temporal.NewApplicationError(errLongHistory, errLongHistory)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) ensureNotDeleted() error {
|
|
if d.deleteDeployment {
|
|
return temporal.NewNonRetryableApplicationError(errDeploymentDeleted, errDeploymentDeleted, nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateCreateWorkerDeployment(args *deploymentspb.CreateWorkerDeploymentArgs) error {
|
|
// Only valid if deployment is deleted or the request ID matches the current one.
|
|
if d.State.GetCreateRequestId() == args.GetRequestId() {
|
|
return nil
|
|
}
|
|
|
|
return temporal.NewNonRetryableApplicationError(errDeploymentAlreadyExists, errDeploymentAlreadyExists, nil)
|
|
}
|
|
|
|
func (d *WorkflowRunner) handleCreateWorkerDeployment(ctx workflow.Context, args *deploymentspb.CreateWorkerDeploymentArgs) (*deploymentspb.CreateWorkerDeploymentResponse, error) {
|
|
// use lock to enforce only one update at a time
|
|
err := d.lock.Lock(ctx)
|
|
if err != nil {
|
|
d.logger.Error("Could not acquire workflow lock")
|
|
return nil, serviceerror.NewDeadlineExceeded("Could not acquire workflow lock")
|
|
}
|
|
defer func() {
|
|
// Even if the update doesn't change the state we mark it as dirty because of created history events.
|
|
d.setStateChanged()
|
|
d.lock.Unlock()
|
|
}()
|
|
|
|
// Re-validate after acquiring lock
|
|
err = d.validateCreateWorkerDeployment(args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if d.State.GetCreateRequestId() == args.GetRequestId() {
|
|
// Duplicate request, return success without writing anything.
|
|
return &deploymentspb.CreateWorkerDeploymentResponse{
|
|
ConflictToken: d.State.ConflictToken,
|
|
}, nil
|
|
}
|
|
|
|
// At this point this a brand-new workflow.
|
|
d.State.LastModifierIdentity = args.GetIdentity()
|
|
d.State.CreateRequestId = args.GetRequestId()
|
|
d.State.ConflictToken, _ = workflow.Now(ctx).MarshalBinary()
|
|
|
|
return &deploymentspb.CreateWorkerDeploymentResponse{
|
|
ConflictToken: d.State.ConflictToken,
|
|
}, nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateCreateWorkerDeploymentVersion(args *deploymentspb.CreateWorkerDeploymentVersionArgs) error {
|
|
if d.deleteDeployment {
|
|
return temporal.NewNonRetryableApplicationError(errDeploymentDeleted, errDeploymentDeleted, nil)
|
|
}
|
|
if existing, ok := d.State.Versions[args.GetVersion()]; ok {
|
|
if existing.GetCreateRequestId() == args.GetRequestId() {
|
|
return nil
|
|
}
|
|
return temporal.NewNonRetryableApplicationError(errVersionAlreadyExists, errVersionAlreadyExists, nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) handleCreateWorkerDeploymentVersion(ctx workflow.Context, args *deploymentspb.CreateWorkerDeploymentVersionArgs) (*deploymentspb.CreateWorkerDeploymentVersionResponse, error) {
|
|
err := d.lock.Lock(ctx)
|
|
if err != nil {
|
|
d.logger.Error("Could not acquire workflow lock")
|
|
return nil, serviceerror.NewDeadlineExceeded("Could not acquire workflow lock")
|
|
}
|
|
defer func() {
|
|
d.setStateChanged()
|
|
d.lock.Unlock()
|
|
}()
|
|
|
|
// Re-validate after acquiring lock.
|
|
err = d.validateCreateWorkerDeploymentVersion(args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Idempotent: version exists with the same request ID.
|
|
if existing, ok := d.State.Versions[args.GetVersion()]; ok && existing.GetCreateRequestId() == args.GetRequestId() {
|
|
return &deploymentspb.CreateWorkerDeploymentVersionResponse{}, nil
|
|
}
|
|
|
|
// Check max versions limit.
|
|
maxVersions := d.getMaxVersions(ctx)
|
|
if len(d.State.Versions) >= maxVersions {
|
|
err := d.tryDeleteVersion(ctx)
|
|
if err != nil {
|
|
return nil, temporal.NewApplicationError(
|
|
fmt.Sprintf("cannot add version %s since maximum number of versions (%d) have been registered in the deployment", args.GetVersion(), maxVersions),
|
|
errTooManyVersions,
|
|
)
|
|
}
|
|
}
|
|
|
|
// Parse version string to get deployment name and build ID.
|
|
versionObj, err := worker_versioning.WorkerDeploymentVersionFromStringV31(args.GetVersion())
|
|
if err != nil {
|
|
return nil, serviceerror.NewInvalidArgument("invalid version string: " + err.Error())
|
|
}
|
|
|
|
// Create or update the Worker Controller Instance for this version.
|
|
computeConfig := args.GetComputeConfig()
|
|
var computeConfigSummary *computepb.ComputeConfigSummary
|
|
if computeConfig != nil {
|
|
updateCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
err = workflow.ExecuteActivity(updateCtx, d.a.UpdateWorkerControllerInstanceFromDeployment, &deploymentspb.UpdateWorkerControllerInstanceInput{
|
|
Version: worker_versioning.ExternalWorkerDeploymentVersionFromVersion(versionObj),
|
|
Identity: args.GetIdentity(),
|
|
UpsertScalingGroups: scalingGroupsToUpsertUpdates(computeConfig.GetScalingGroups()),
|
|
}).Get(ctx, &computeConfigSummary)
|
|
if err != nil {
|
|
var appErr *temporal.ApplicationError
|
|
if errors.As(err, &appErr) && appErr.Type() == errInvalidComputeConfig {
|
|
return nil, appErr
|
|
}
|
|
return nil, serviceerror.NewInternalf("update worker controller instance: %v", err)
|
|
}
|
|
}
|
|
|
|
// Start the version workflow via activity.
|
|
activityCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
err = workflow.ExecuteActivity(activityCtx, d.a.StartWorkerDeploymentVersionWorkflow, &deploymentspb.StartWorkerDeploymentVersionRequest{
|
|
DeploymentName: versionObj.DeploymentName,
|
|
BuildId: versionObj.BuildId,
|
|
RequestId: args.GetRequestId(),
|
|
Identity: args.GetIdentity(),
|
|
ComputeConfig: computeConfigSummary,
|
|
}).Get(ctx, nil)
|
|
if err != nil {
|
|
if computeConfig != nil {
|
|
deleteCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
deleteErr := workflow.ExecuteActivity(deleteCtx, d.a.DeleteWorkerControllerInstanceFromDeployment, &deploymentspb.DeleteWorkerControllerInstanceInput{
|
|
Version: worker_versioning.ExternalWorkerDeploymentVersionFromVersion(versionObj),
|
|
Identity: args.GetIdentity(),
|
|
}).Get(ctx, nil)
|
|
if deleteErr != nil {
|
|
d.logger.Warn("Failed to delete worker controller instance", "error", err)
|
|
}
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
// Add version to local state.
|
|
d.State.Versions[args.GetVersion()] = &deploymentspb.WorkerDeploymentVersionSummary{
|
|
Version: args.GetVersion(),
|
|
CreateTime: timestamppb.New(workflow.Now(ctx)),
|
|
Status: enumspb.WORKER_DEPLOYMENT_VERSION_STATUS_CREATED,
|
|
CreateRequestId: args.GetRequestId(),
|
|
ComputeConfig: computeConfigSummary,
|
|
}
|
|
d.metrics.Counter(metrics.WorkerDeploymentVersionCreated.Name()).Inc(1)
|
|
|
|
if err := d.updateMemo(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
return &deploymentspb.CreateWorkerDeploymentVersionResponse{}, nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) addVersionToWorkerDeployment(ctx workflow.Context, args *deploymentspb.AddVersionUpdateArgs) error {
|
|
if d.State.Versions == nil {
|
|
return nil
|
|
}
|
|
|
|
for _, k := range workflow.DeterministicKeys(d.State.Versions) {
|
|
v := d.State.Versions[k]
|
|
if v.Version == args.Version {
|
|
return nil
|
|
}
|
|
}
|
|
|
|
maxVersions := d.getMaxVersions(ctx)
|
|
|
|
if len(d.State.Versions) >= maxVersions {
|
|
err := d.tryDeleteVersion(ctx)
|
|
if err != nil {
|
|
return temporal.NewApplicationError(fmt.Sprintf("cannot add version %s since maximum number of versions (%d) have been registered in the deployment", args.Version, maxVersions), errTooManyVersions)
|
|
}
|
|
}
|
|
|
|
d.State.Versions[args.Version] = &deploymentspb.WorkerDeploymentVersionSummary{
|
|
Version: args.Version,
|
|
CreateTime: args.CreateTime,
|
|
Status: enumspb.WORKER_DEPLOYMENT_VERSION_STATUS_INACTIVE,
|
|
}
|
|
d.metrics.Counter(metrics.WorkerDeploymentVersionCreated.Name()).Inc(1)
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) handleRegisterWorker(ctx workflow.Context, args *deploymentspb.RegisterWorkerInWorkerDeploymentArgs) error {
|
|
// TODO: there is a small race condition where the deployment is just deleted and got a register update before closing itself.
|
|
// In that case, we should ideally not reject the update, but revive the workflow so that the caller does not need to retry.
|
|
// In practice this should be fine because the polls will retry and Deployment workflows are short-lived.
|
|
// Same principle applies for Version workflows, but they can be slightly more long-lived of they are handling long propagations.
|
|
// Hence, the revive logic is implemented in Version workflow but not here yet.
|
|
if err := d.ensureNotDeleted(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// use lock to enforce only one update at a time
|
|
err := d.lock.Lock(ctx)
|
|
if err != nil {
|
|
d.logger.Error("Could not acquire workflow lock")
|
|
return serviceerror.NewDeadlineExceeded("Could not acquire workflow lock")
|
|
}
|
|
defer func() {
|
|
// Even if the update doesn't change the state we mark it as dirty because of created history events.
|
|
d.setStateChanged()
|
|
d.lock.Unlock()
|
|
}()
|
|
|
|
version := worker_versioning.WorkerDeploymentVersionToStringV31(args.Version)
|
|
|
|
// Add version to local state of the workflow, if not already present.
|
|
err = d.addVersionToWorkerDeployment(ctx, &deploymentspb.AddVersionUpdateArgs{
|
|
Version: version,
|
|
CreateTime: timestamppb.New(workflow.Now(ctx)),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var routingConfigToSync *deploymentpb.RoutingConfig
|
|
if d.hasMinVersion(AsyncSetCurrentAndRamping) {
|
|
routingConfigToSync = d.GetState().GetRoutingConfig()
|
|
}
|
|
|
|
// Register task-queue worker in version workflow.
|
|
activityCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
err = workflow.ExecuteActivity(activityCtx, d.a.RegisterWorkerInVersion, &deploymentspb.RegisterWorkerInVersionArgs{
|
|
TaskQueueName: args.TaskQueueName,
|
|
TaskQueueType: args.TaskQueueType,
|
|
MaxTaskQueues: args.MaxTaskQueues,
|
|
Version: version,
|
|
RoutingConfig: routingConfigToSync,
|
|
}).Get(ctx, nil)
|
|
if err != nil {
|
|
var appError *temporal.ApplicationError
|
|
if errors.As(err, &appError) {
|
|
if appError.Type() == errMaxTaskQueuesInVersionType {
|
|
return temporal.NewApplicationError(
|
|
fmt.Sprintf("cannot add task queue %v since maximum number of task queues (%d) have been registered in deployment", args.TaskQueueName, args.MaxTaskQueues),
|
|
errMaxTaskQueuesInVersionType,
|
|
)
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
if d.State.Versions[version].Status == enumspb.WORKER_DEPLOYMENT_VERSION_STATUS_CREATED {
|
|
// now that a poller is seen, we should update the status to INACTIVE
|
|
d.State.Versions[version].Status = enumspb.WORKER_DEPLOYMENT_VERSION_STATUS_INACTIVE
|
|
}
|
|
|
|
// update memo
|
|
return d.updateMemo(ctx)
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateDeleteDeployment() error {
|
|
if len(d.State.Versions) > 0 {
|
|
return serviceerror.NewFailedPrecondition("deployment has versions, can't be deleted")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) handleDeleteDeployment(ctx workflow.Context) error {
|
|
// use lock to enforce only one update at a time
|
|
err := d.lock.Lock(ctx)
|
|
if err != nil {
|
|
d.logger.Error("Could not acquire workflow lock")
|
|
return serviceerror.NewDeadlineExceeded("Could not acquire workflow lock")
|
|
}
|
|
defer func() {
|
|
// Even if the update doesn't change the state we mark it as dirty because of created history events.
|
|
d.setStateChanged()
|
|
d.lock.Unlock()
|
|
}()
|
|
|
|
if len(d.State.Versions) == 0 {
|
|
d.deleteDeployment = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) rampingVersionStringUnversioned(s string) bool {
|
|
return s == worker_versioning.UnversionedVersionId || s == ""
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateStateBeforeAcceptingRampingUpdate(args *deploymentspb.SetRampingVersionArgs) error {
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
if args.Version == d.State.GetRoutingConfig().GetRampingVersion() &&
|
|
args.Percentage == d.State.GetRoutingConfig().GetRampingVersionPercentage() &&
|
|
args.Identity == d.State.GetLastModifierIdentity() {
|
|
return temporal.NewApplicationError("version already ramping, no change", errNoChangeType, d.State.GetConflictToken())
|
|
}
|
|
|
|
if args.ConflictToken != nil && !bytes.Equal(args.ConflictToken, d.State.GetConflictToken()) {
|
|
return temporal.NewApplicationError("conflict token mismatch", errFailedPrecondition)
|
|
}
|
|
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
if args.Version == d.State.GetRoutingConfig().GetCurrentVersion() &&
|
|
!(args.Version == worker_versioning.UnversionedVersionId && args.Percentage == 0) {
|
|
d.logger.Info("version can't be set to ramping since it is already current")
|
|
return temporal.NewApplicationError(fmt.Sprintf("requested ramping version %s is already current", args.Version), errFailedPrecondition)
|
|
}
|
|
|
|
if _, ok := d.State.GetVersions()[args.Version]; !ok &&
|
|
args.Version != worker_versioning.UnversionedVersionId &&
|
|
args.Version != "" &&
|
|
!args.GetAllowNoPollers() {
|
|
d.logger.Info("version not found in deployment")
|
|
return temporal.NewApplicationError(fmt.Sprintf("requested ramping version %s not found in deployment", args.Version), errVersionNotFound)
|
|
}
|
|
|
|
if d.State.ManagerIdentity != "" && d.State.ManagerIdentity != args.Identity {
|
|
return serviceerror.NewFailedPrecondition(fmt.Sprintf(ErrManagerIdentityMismatch, d.State.ManagerIdentity, args.Identity))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateSetRampingVersion(args *deploymentspb.SetRampingVersionArgs) error {
|
|
if err := d.ensureNotDeleted(); err != nil {
|
|
return err
|
|
}
|
|
return d.validateStateBeforeAcceptingRampingUpdate(args)
|
|
}
|
|
|
|
//revive:disable-next-line:cognitive-complexity
|
|
//nolint:staticcheck // deprecated stuff will be cleaned
|
|
func (d *WorkflowRunner) handleSetRampingVersion(ctx workflow.Context, args *deploymentspb.SetRampingVersionArgs) (*deploymentspb.SetRampingVersionResponse, error) {
|
|
if err := d.preUpdateChecks(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// use lock to enforce only one update at a time
|
|
err := d.lock.Lock(ctx)
|
|
if err != nil {
|
|
d.logger.Error("Could not acquire workflow lock")
|
|
return nil, serviceerror.NewDeadlineExceeded("Could not acquire workflow lock")
|
|
}
|
|
defer func() {
|
|
// Even if the update doesn't change the state we mark it as dirty because of created history events.
|
|
d.setStateChanged()
|
|
d.lock.Unlock()
|
|
}()
|
|
|
|
// Validating the state before starting the SetRampingVersion operation. This is required due to the following reason:
|
|
// The validator accepts/rejects updates based on the state of the deployment workflow. Theoretically, two concurrent update requests
|
|
// might be accepted by the validator since the state of the workflow, at that point in time, is valid for the updates to take place. Since this update handler
|
|
// enforces sequential updates, after the first update completes, the local state of the deployment workflow will change. The second update,
|
|
// now already accepted by the validator, should now not be allowed to run since the state of the workflow is different.
|
|
err = d.validateStateBeforeAcceptingRampingUpdate(args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
prevRampingVersion := d.State.RoutingConfig.RampingVersion
|
|
prevRampingVersionPercentage := d.State.RoutingConfig.RampingVersionPercentage
|
|
|
|
newRampingVersion := args.Version
|
|
routingUpdateTime := timestamppb.New(workflow.Now(ctx))
|
|
|
|
if _, ok := d.State.Versions[args.Version]; !ok &&
|
|
args.Version != worker_versioning.UnversionedVersionId &&
|
|
args.Version != "" &&
|
|
args.GetAllowNoPollers() {
|
|
d.logger.Info("version not found in deployment, but AllowNoPollers is true, so we will create the version")
|
|
if err := d.addVersionToWorkerDeployment(ctx, &deploymentspb.AddVersionUpdateArgs{Version: newRampingVersion, CreateTime: routingUpdateTime}); err != nil {
|
|
return nil, err // only possible error is errTooManyVersions
|
|
}
|
|
v, err := worker_versioning.WorkerDeploymentVersionFromStringV31(newRampingVersion)
|
|
if err != nil {
|
|
return nil, err // this would never happen, because version string formatting was already checked earlier
|
|
}
|
|
if err := d.startVersion(ctx, &deploymentspb.StartWorkerDeploymentVersionRequest{
|
|
DeploymentName: v.GetDeploymentName(),
|
|
BuildId: v.GetBuildId(),
|
|
RequestId: d.newUUID(ctx),
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
var rampingSinceTime *timestamppb.Timestamp
|
|
var rampingVersionUpdateTime *timestamppb.Timestamp
|
|
|
|
asyncMode := d.hasMinVersion(AsyncSetCurrentAndRamping)
|
|
|
|
if prevRampingVersion != newRampingVersion || prevRampingVersionPercentage != args.Percentage || !asyncMode {
|
|
// In async mode we do not touch routing config and versions if this is not changing the
|
|
// ramping version or percentage (but could still come here because modifier identity is changing).
|
|
|
|
// Determine timestamps based on whether we're setting or unsetting ramp
|
|
if newRampingVersion == "" {
|
|
// unsetting ramp
|
|
rampingVersionUpdateTime = routingUpdateTime
|
|
} else if prevRampingVersion == newRampingVersion {
|
|
// version was already ramping, user changing ramp %
|
|
rampingSinceTime = d.State.RoutingConfig.RampingVersionChangedTime
|
|
rampingVersionUpdateTime = d.State.RoutingConfig.RampingVersionChangedTime
|
|
} else {
|
|
// version ramping for the first time
|
|
rampingSinceTime = routingUpdateTime
|
|
rampingVersionUpdateTime = routingUpdateTime
|
|
}
|
|
|
|
// Build pending routing config with the updated ramping version
|
|
// Initialize for both sync and async modes to simplify state update logic
|
|
// Ensure CurrentDeploymentVersion is populated from deprecated field if needed for backward compatibility
|
|
currentDeploymentVersion := d.State.RoutingConfig.CurrentDeploymentVersion
|
|
if currentDeploymentVersion == nil {
|
|
currentDeploymentVersion = worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(d.State.RoutingConfig.CurrentVersion)
|
|
}
|
|
pendingRoutingConfig := &deploymentpb.RoutingConfig{
|
|
CurrentDeploymentVersion: currentDeploymentVersion,
|
|
CurrentVersion: d.State.RoutingConfig.CurrentVersion,
|
|
RampingDeploymentVersion: worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(newRampingVersion),
|
|
RampingVersion: newRampingVersion,
|
|
RampingVersionPercentage: args.Percentage,
|
|
CurrentVersionChangedTime: d.State.RoutingConfig.CurrentVersionChangedTime,
|
|
RampingVersionChangedTime: rampingVersionUpdateTime,
|
|
RampingVersionPercentageChangedTime: routingUpdateTime,
|
|
RevisionNumber: d.State.RoutingConfig.RevisionNumber,
|
|
}
|
|
|
|
var routingConfigToSync *deploymentpb.RoutingConfig
|
|
|
|
if asyncMode {
|
|
pendingRoutingConfig.RevisionNumber++
|
|
// only setting it in the request if it's async mode
|
|
routingConfigToSync = pendingRoutingConfig
|
|
}
|
|
|
|
if newRampingVersion == "" {
|
|
err = d.unsetRamp(ctx, routingUpdateTime, routingConfigToSync, prevRampingVersion, asyncMode, pendingRoutingConfig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
err = d.setRamp(ctx,
|
|
args,
|
|
prevRampingVersion,
|
|
newRampingVersion,
|
|
routingUpdateTime,
|
|
rampingSinceTime,
|
|
routingConfigToSync,
|
|
asyncMode,
|
|
pendingRoutingConfig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// update local state - use pendingRoutingConfig (initialized for both sync and async modes)
|
|
d.State.RoutingConfig = pendingRoutingConfig
|
|
}
|
|
|
|
d.State.ConflictToken, _ = routingUpdateTime.AsTime().MarshalBinary()
|
|
d.State.LastModifierIdentity = args.Identity
|
|
|
|
// update memo
|
|
if err = d.updateMemo(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &deploymentspb.SetRampingVersionResponse{
|
|
PreviousVersion: prevRampingVersion,
|
|
PreviousPercentage: prevRampingVersionPercentage,
|
|
ConflictToken: d.State.ConflictToken,
|
|
}, nil
|
|
|
|
}
|
|
|
|
//nolint:staticcheck // SA1019
|
|
func (d *WorkflowRunner) unsetRamp(
|
|
ctx workflow.Context,
|
|
routingUpdateTime *timestamppb.Timestamp,
|
|
routingConfigToSync *deploymentpb.RoutingConfig,
|
|
prevRampingVersion string,
|
|
asyncMode bool,
|
|
pendingRoutingConfig *deploymentpb.RoutingConfig,
|
|
) error {
|
|
unsetRampUpdateArgs := &deploymentspb.SyncVersionStateUpdateArgs{
|
|
RoutingUpdateTime: routingUpdateTime,
|
|
RampingSinceTime: nil, // remove ramp
|
|
RampPercentage: 0, // remove ramp
|
|
RoutingConfig: routingConfigToSync,
|
|
}
|
|
|
|
if !d.rampingVersionStringUnversioned(prevRampingVersion) {
|
|
if _, err := d.syncVersion(ctx, prevRampingVersion, unsetRampUpdateArgs); err != nil {
|
|
return err
|
|
}
|
|
} else if asyncMode {
|
|
// Here, we are unsetting unversioned ramp in async mode. This only can happen if there IS a current version, so we
|
|
// propagate the ramp status in routing config through the current version.
|
|
if _, err := d.syncVersion(ctx, pendingRoutingConfig.CurrentVersion, unsetRampUpdateArgs); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
// Only should call this in sync mode
|
|
if err := d.syncUnversionedRamp(ctx, unsetRampUpdateArgs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Set summary drainage status immediately to draining.
|
|
// We know prevRampingVersion cannot have been current, so it must now be draining
|
|
d.setDrainageStatus(prevRampingVersion, enumspb.VERSION_DRAINAGE_STATUS_DRAINING, routingUpdateTime)
|
|
return nil
|
|
}
|
|
|
|
//nolint:staticcheck // SA1019
|
|
func (d *WorkflowRunner) setRamp(
|
|
ctx workflow.Context,
|
|
args *deploymentspb.SetRampingVersionArgs,
|
|
prevRampingVersion string,
|
|
newRampingVersion string,
|
|
routingUpdateTime *timestamppb.Timestamp,
|
|
rampingSinceTime *timestamppb.Timestamp,
|
|
routingConfigToSync *deploymentpb.RoutingConfig,
|
|
asyncMode bool,
|
|
pendingRoutingConfig *deploymentpb.RoutingConfig,
|
|
) error {
|
|
if prevRampingVersion != newRampingVersion {
|
|
// version ramping for the first time - need to check for missing task queues
|
|
|
|
currentVersion := d.State.RoutingConfig.CurrentVersion
|
|
if !args.IgnoreMissingTaskQueues &&
|
|
currentVersion != worker_versioning.UnversionedVersionId &&
|
|
newRampingVersion != worker_versioning.UnversionedVersionId {
|
|
isMissingTaskQueues, err := d.isVersionMissingTaskQueues(ctx, currentVersion, newRampingVersion)
|
|
if err != nil {
|
|
d.logger.Info("Error verifying poller presence in version", "error", err)
|
|
return err
|
|
}
|
|
if isMissingTaskQueues {
|
|
newRampingVersionObj, _ := worker_versioning.WorkerDeploymentVersionFromStringV31(newRampingVersion)
|
|
return serviceerror.NewFailedPreconditionf(
|
|
ErrRampingVersionDoesNotHaveAllTaskQueues,
|
|
worker_versioning.WorkerDeploymentVersionToStringV32(newRampingVersionObj),
|
|
)
|
|
}
|
|
}
|
|
if !asyncMode {
|
|
// Erase summary drainage status immediately, so it is not draining/drained.
|
|
d.setDrainageStatus(newRampingVersion, enumspb.VERSION_DRAINAGE_STATUS_UNSPECIFIED, routingUpdateTime)
|
|
}
|
|
}
|
|
|
|
setRampUpdateArgs := &deploymentspb.SyncVersionStateUpdateArgs{
|
|
RoutingUpdateTime: routingUpdateTime,
|
|
RampingSinceTime: rampingSinceTime,
|
|
RampPercentage: args.Percentage,
|
|
RoutingConfig: routingConfigToSync,
|
|
}
|
|
if !d.rampingVersionStringUnversioned(newRampingVersion) {
|
|
if _, err := d.syncVersion(ctx, newRampingVersion, setRampUpdateArgs); err != nil {
|
|
return err
|
|
}
|
|
} else if asyncMode {
|
|
// Here, we are setting unversioned ramp in async mode. This only can happen if there IS a current version, so we
|
|
// propagate the ramp status in routing config through the current version.
|
|
if _, err := d.syncVersion(ctx, pendingRoutingConfig.CurrentVersion, setRampUpdateArgs); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
// Only should call this in sync mode
|
|
if err := d.syncUnversionedRamp(ctx, setRampUpdateArgs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// tell previous ramping version, if present, that it's no longer ramping
|
|
if prevRampingVersion != "" && prevRampingVersion != newRampingVersion {
|
|
commitRoutingFirst := workflow.GetVersion(ctx, "commit-routing-first", workflow.DefaultVersion, 0) >= 0
|
|
if asyncMode && commitRoutingFirst {
|
|
// Commit routing config before demoting prev ramp to avoid partial state on failure.
|
|
d.State.RoutingConfig = pendingRoutingConfig
|
|
// Signal the prev ramp version workflow to demote. Unversioned ramp has no version
|
|
// workflow and is already handled by the new ramp version's sync (step 1).
|
|
if !d.rampingVersionStringUnversioned(prevRampingVersion) {
|
|
d.signalDemoteVersion(ctx, prevRampingVersion, pendingRoutingConfig)
|
|
}
|
|
d.setVersionSummaryDraining(prevRampingVersion, routingUpdateTime)
|
|
} else {
|
|
err := d.unsetPreviousRamp(ctx, routingUpdateTime, routingConfigToSync, prevRampingVersion, asyncMode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
//nolint:staticcheck // SA1019
|
|
func (d *WorkflowRunner) unsetPreviousRamp(
|
|
ctx workflow.Context,
|
|
routingUpdateTime *timestamppb.Timestamp,
|
|
routingConfigToSync *deploymentpb.RoutingConfig,
|
|
prevRampingVersion string,
|
|
asyncMode bool,
|
|
) error {
|
|
unsetRampUpdateArgs := &deploymentspb.SyncVersionStateUpdateArgs{
|
|
RoutingUpdateTime: routingUpdateTime,
|
|
RampingSinceTime: nil, // remove ramp
|
|
RampPercentage: 0, // remove ramp
|
|
RoutingConfig: routingConfigToSync,
|
|
}
|
|
if !d.rampingVersionStringUnversioned(prevRampingVersion) {
|
|
if _, err := d.syncVersion(ctx, prevRampingVersion, unsetRampUpdateArgs); err != nil {
|
|
return err
|
|
}
|
|
} else if asyncMode {
|
|
// Here, we are setting a versioned ramp on top of an unversioned ramp in async mode. In this case we
|
|
// already synced the ramp status through the new ramping version so there is no need for another sync.
|
|
} else {
|
|
// Only should call this in sync mode
|
|
if err := d.syncUnversionedRamp(ctx, unsetRampUpdateArgs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if !asyncMode {
|
|
// Set summary drainage status immediately to draining.
|
|
// We know prevRampingVersion cannot have been current, so it must now be draining
|
|
d.setDrainageStatus(prevRampingVersion, enumspb.VERSION_DRAINAGE_STATUS_DRAINING, routingUpdateTime)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) setDrainageStatus(version string, status enumspb.VersionDrainageStatus, routingUpdateTime *timestamppb.Timestamp) {
|
|
if summary := d.State.GetVersions()[version]; summary != nil {
|
|
summary.DrainageStatus = status
|
|
summary.DrainageInfo = &deploymentpb.VersionDrainageInfo{
|
|
Status: status,
|
|
LastChangedTime: routingUpdateTime,
|
|
LastCheckedTime: routingUpdateTime,
|
|
}
|
|
}
|
|
}
|
|
|
|
func (d *WorkflowRunner) setVersionSummaryDraining(version string, routingUpdateTime *timestamppb.Timestamp) {
|
|
if summary := d.State.GetVersions()[version]; summary != nil {
|
|
summary.Status = enumspb.WORKER_DEPLOYMENT_VERSION_STATUS_DRAINING
|
|
summary.CurrentSinceTime = nil
|
|
summary.RampingSinceTime = nil
|
|
summary.RoutingUpdateTime = routingUpdateTime
|
|
summary.LastDeactivationTime = routingUpdateTime
|
|
d.setDrainageStatus(version, enumspb.VERSION_DRAINAGE_STATUS_DRAINING, routingUpdateTime)
|
|
}
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateDeleteVersion(args *deploymentspb.DeleteVersionArgs) error {
|
|
if err := d.ensureNotDeleted(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, ok := d.State.Versions[args.Version]; !ok {
|
|
return temporal.NewApplicationError("version not found in deployment", errVersionNotFound)
|
|
}
|
|
|
|
// Check if the version is not current or ramping. This condition is better to be checked in the
|
|
// deployment workflow because that's the source of truth for routing config.
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
if d.State.RoutingConfig.CurrentVersion == args.Version || d.State.RoutingConfig.RampingVersion == args.Version {
|
|
versionObj, _ := worker_versioning.WorkerDeploymentVersionFromStringV31(args.Version)
|
|
// activity won't retry on this error since version not eligible for deletion
|
|
return serviceerror.NewFailedPreconditionf(ErrVersionIsCurrentOrRamping, worker_versioning.WorkerDeploymentVersionToStringV32(versionObj))
|
|
}
|
|
|
|
// Ignore the manager identity check if the delete operation is initiated by the server internally
|
|
if !args.GetServerDelete() && d.State.ManagerIdentity != "" && d.State.ManagerIdentity != args.Identity {
|
|
return serviceerror.NewFailedPreconditionf(ErrManagerIdentityMismatch, d.State.ManagerIdentity, args.Identity)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) deleteVersion(ctx workflow.Context, args *deploymentspb.DeleteVersionArgs) error {
|
|
// ask version to delete itself
|
|
activityCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
var res deploymentspb.SyncVersionStateActivityResult
|
|
err := workflow.ExecuteActivity(activityCtx, d.a.DeleteWorkerDeploymentVersion, &deploymentspb.DeleteVersionActivityArgs{
|
|
Identity: args.Identity,
|
|
DeploymentName: d.DeploymentName,
|
|
Version: args.Version,
|
|
RequestId: uuid.NewString(),
|
|
SkipDrainage: args.SkipDrainage,
|
|
AsyncPropagation: d.hasMinVersion(AsyncSetCurrentAndRamping),
|
|
}).Get(ctx, &res)
|
|
if err != nil {
|
|
var activityError *temporal.ActivityError
|
|
var applicationError *temporal.ApplicationError
|
|
if errors.As(err, &activityError) && errors.As(activityError.Unwrap(), &applicationError) &&
|
|
(applicationError.Type() == errVersionHasPollers || applicationError.Type() == errVersionIsDraining) {
|
|
return serviceerror.NewFailedPrecondition(applicationError.Message())
|
|
}
|
|
return err
|
|
}
|
|
// update local state
|
|
delete(d.State.Versions, args.Version)
|
|
// remove from propagating versions if it's there
|
|
delete(d.State.PropagatingRevisions, worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(args.GetVersion()).GetBuildId())
|
|
if !args.GetServerDelete() {
|
|
d.State.LastModifierIdentity = args.Identity
|
|
}
|
|
// update memo
|
|
return d.updateMemo(ctx)
|
|
}
|
|
|
|
func (d *WorkflowRunner) handleDeleteVersion(ctx workflow.Context, args *deploymentspb.DeleteVersionArgs) error {
|
|
if err := d.preUpdateChecks(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
// use lock to enforce only one update at a time
|
|
err := d.lock.Lock(ctx)
|
|
if err != nil {
|
|
d.logger.Error("Could not acquire workflow lock")
|
|
return serviceerror.NewDeadlineExceeded("Could not acquire workflow lock")
|
|
}
|
|
defer func() {
|
|
// Even if the update doesn't change the state we mark it as dirty because of created history events.
|
|
d.setStateChanged()
|
|
d.lock.Unlock()
|
|
}()
|
|
|
|
// Validating the state before starting the DeleteVersion operation. This is required due to the following reason:
|
|
// The validator accepts/rejects updates based on the state of the deployment workflow. Theoretically, two concurrent delete version requests
|
|
// might be accepted by the validator since the local state of the workflow contains the version which is requested to be deleted. Since this update handler
|
|
// enforces sequential updates, after the first update completes, the version will be removed from the local state of the deployment workflow. The second update,
|
|
// now already accepted by the validator, should now not be allowed to run since the initial workflow state is different.
|
|
err = d.validateDeleteVersion(args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return d.deleteVersion(ctx, args)
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateStateBeforeAcceptingSetManager(args *deploymentspb.SetManagerIdentityArgs) error {
|
|
if d.State.GetManagerIdentity() == args.ManagerIdentity && d.State.GetLastModifierIdentity() == args.Identity {
|
|
return temporal.NewApplicationError("no change", errNoChangeType, d.State.ConflictToken)
|
|
}
|
|
if args.ConflictToken != nil && !bytes.Equal(args.ConflictToken, d.State.ConflictToken) {
|
|
return temporal.NewApplicationError("conflict token mismatch", errFailedPrecondition)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateSetManager(args *deploymentspb.SetManagerIdentityArgs) error {
|
|
if err := d.ensureNotDeleted(); err != nil {
|
|
return err
|
|
}
|
|
return d.validateStateBeforeAcceptingSetManager(args)
|
|
}
|
|
|
|
func (d *WorkflowRunner) handleSetManager(ctx workflow.Context, args *deploymentspb.SetManagerIdentityArgs) (*deploymentspb.SetManagerIdentityResponse, error) {
|
|
if err := d.preUpdateChecks(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// use lock to enforce only one update at a time
|
|
err := d.lock.Lock(ctx)
|
|
if err != nil {
|
|
d.logger.Error("Could not acquire workflow lock")
|
|
return nil, serviceerror.NewDeadlineExceeded("Could not acquire workflow lock")
|
|
}
|
|
defer func() {
|
|
// Even if the update doesn't change the state we mark it as dirty because of created history events.
|
|
d.setStateChanged()
|
|
d.lock.Unlock()
|
|
}()
|
|
|
|
err = d.validateStateBeforeAcceptingSetManager(args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
prevManager := d.State.ManagerIdentity
|
|
|
|
// update local state
|
|
d.State.ManagerIdentity = args.ManagerIdentity
|
|
d.State.LastModifierIdentity = args.Identity
|
|
d.State.ConflictToken, _ = workflow.Now(ctx).MarshalBinary()
|
|
|
|
// no need to update memo because identity and manager identity are not in it
|
|
return &deploymentspb.SetManagerIdentityResponse{
|
|
PreviousManagerIdentity: prevManager,
|
|
ConflictToken: d.State.ConflictToken,
|
|
}, nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateStateBeforeAcceptingSetCurrent(args *deploymentspb.SetCurrentVersionArgs) error {
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
if d.State.GetRoutingConfig().GetCurrentVersion() == args.Version && d.State.GetLastModifierIdentity() == args.Identity {
|
|
return temporal.NewApplicationError("no change", errNoChangeType, d.State.ConflictToken)
|
|
}
|
|
if args.ConflictToken != nil && !bytes.Equal(args.ConflictToken, d.State.ConflictToken) {
|
|
return temporal.NewApplicationError("conflict token mismatch", errFailedPrecondition)
|
|
}
|
|
if _, ok := d.State.Versions[args.Version]; !ok &&
|
|
args.Version != worker_versioning.UnversionedVersionId &&
|
|
!args.GetAllowNoPollers() {
|
|
d.logger.Info("version not found in deployment")
|
|
return temporal.NewApplicationError(fmt.Sprintf("version %s not found in deployment", args.Version), errVersionNotFound)
|
|
}
|
|
if d.State.ManagerIdentity != "" && d.State.ManagerIdentity != args.Identity {
|
|
return serviceerror.NewFailedPrecondition(fmt.Sprintf(ErrManagerIdentityMismatch, d.State.ManagerIdentity, args.Identity))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) validateSetCurrent(args *deploymentspb.SetCurrentVersionArgs) error {
|
|
if err := d.ensureNotDeleted(); err != nil {
|
|
return err
|
|
}
|
|
|
|
return d.validateStateBeforeAcceptingSetCurrent(args)
|
|
}
|
|
|
|
//nolint:staticcheck // deprecated stuff will be cleaned
|
|
func (d *WorkflowRunner) handleSetCurrent(ctx workflow.Context, args *deploymentspb.SetCurrentVersionArgs) (*deploymentspb.SetCurrentVersionResponse, error) {
|
|
if err := d.preUpdateChecks(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// use lock to enforce only one update at a time
|
|
err := d.lock.Lock(ctx)
|
|
if err != nil {
|
|
d.logger.Error("Could not acquire workflow lock")
|
|
return nil, serviceerror.NewDeadlineExceeded("Could not acquire workflow lock")
|
|
}
|
|
defer func() {
|
|
// Even if the update doesn't change the state we mark it as dirty because of created history events.
|
|
d.setStateChanged()
|
|
d.lock.Unlock()
|
|
}()
|
|
|
|
// Log state before update
|
|
// TODO(carlydf): remove verbose logging
|
|
d.logger.Info("Starting SetCurrent update",
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
"current_version", d.State.GetRoutingConfig().GetCurrentVersion(),
|
|
"new_version", args.Version,
|
|
"routing_config", d.State.GetRoutingConfig())
|
|
|
|
// Validating the state before starting the SetCurrent operation. This is required due to the following reason:
|
|
// The validator accepts/rejects updates based on the state of the deployment workflow. Theoretically, two concurrent update requests
|
|
// might be accepted by the validator since the state of the workflow, at that point in time, is valid for the updates to take place. Since this update handler
|
|
// enforces sequential updates, after the first update completes, the local state of the deployment workflow will change. The second update,
|
|
// now already accepted by the validator, should now not be allowed to run since the state of the workflow is different.
|
|
err = d.validateStateBeforeAcceptingSetCurrent(args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
prevCurrentVersion := d.State.RoutingConfig.CurrentVersion
|
|
newCurrentVersion := args.Version
|
|
updateTime := timestamppb.New(workflow.Now(ctx))
|
|
|
|
if _, ok := d.State.Versions[args.Version]; !ok &&
|
|
args.Version != worker_versioning.UnversionedVersionId &&
|
|
args.GetAllowNoPollers() {
|
|
d.logger.Info("version not found in deployment, but AllowNoPollers is true, so we will create the version")
|
|
if err := d.addVersionToWorkerDeployment(ctx, &deploymentspb.AddVersionUpdateArgs{Version: newCurrentVersion, CreateTime: updateTime}); err != nil {
|
|
return nil, err // only possible error is errTooManyVersions
|
|
}
|
|
v, err := worker_versioning.WorkerDeploymentVersionFromStringV31(newCurrentVersion)
|
|
if err != nil {
|
|
return nil, err // this would never happen, because version string formatting was already checked earlier
|
|
}
|
|
if err := d.startVersion(ctx, &deploymentspb.StartWorkerDeploymentVersionRequest{
|
|
DeploymentName: v.GetDeploymentName(),
|
|
BuildId: v.GetBuildId(),
|
|
RequestId: d.newUUID(ctx),
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if !args.IgnoreMissingTaskQueues &&
|
|
prevCurrentVersion != worker_versioning.UnversionedVersionId &&
|
|
newCurrentVersion != worker_versioning.UnversionedVersionId {
|
|
isMissingTaskQueues, err := d.isVersionMissingTaskQueues(ctx, prevCurrentVersion, newCurrentVersion)
|
|
if err != nil {
|
|
d.logger.Info("Error verifying poller presence in version", "error", err)
|
|
return nil, err
|
|
}
|
|
if isMissingTaskQueues {
|
|
newCurrentVersionObj, _ := worker_versioning.WorkerDeploymentVersionFromStringV31(newCurrentVersion)
|
|
return nil, serviceerror.NewFailedPreconditionf(
|
|
ErrCurrentVersionDoesNotHaveAllTaskQueues,
|
|
worker_versioning.WorkerDeploymentVersionToStringV32(newCurrentVersionObj),
|
|
)
|
|
}
|
|
}
|
|
|
|
asyncMode := d.hasMinVersion(AsyncSetCurrentAndRamping)
|
|
|
|
if prevCurrentVersion != newCurrentVersion || !asyncMode {
|
|
// In async mode we do not touch routing config and versions if this is not changing the
|
|
// current version (but could still come here because modifier identity is changing).
|
|
|
|
// Build pending routing config with the updated current version
|
|
// Initialize for both sync and async modes to simplify state update logic
|
|
// Ensure RampingDeploymentVersion is populated from deprecated field if needed for backward compatibility
|
|
rampingDeploymentVersion := d.State.RoutingConfig.RampingDeploymentVersion
|
|
if rampingDeploymentVersion == nil {
|
|
rampingDeploymentVersion = worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(d.State.RoutingConfig.RampingVersion)
|
|
}
|
|
pendingRoutingConfig := &deploymentpb.RoutingConfig{
|
|
CurrentDeploymentVersion: worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(newCurrentVersion),
|
|
CurrentVersion: newCurrentVersion,
|
|
RampingDeploymentVersion: rampingDeploymentVersion,
|
|
RampingVersion: d.State.RoutingConfig.RampingVersion,
|
|
RampingVersionPercentage: d.State.RoutingConfig.RampingVersionPercentage,
|
|
CurrentVersionChangedTime: updateTime,
|
|
RampingVersionChangedTime: d.State.RoutingConfig.RampingVersionChangedTime,
|
|
RampingVersionPercentageChangedTime: d.State.RoutingConfig.RampingVersionPercentageChangedTime,
|
|
RevisionNumber: d.State.RoutingConfig.RevisionNumber,
|
|
}
|
|
|
|
var routingConfigToSync *deploymentpb.RoutingConfig
|
|
if asyncMode {
|
|
pendingRoutingConfig.RevisionNumber++
|
|
// only setting it in the request if it's async mode
|
|
routingConfigToSync = pendingRoutingConfig
|
|
}
|
|
|
|
// Unset ramping if it's being promoted to current
|
|
if newCurrentVersion == d.State.RoutingConfig.RampingVersion {
|
|
pendingRoutingConfig.RampingVersion = ""
|
|
pendingRoutingConfig.RampingDeploymentVersion = nil
|
|
pendingRoutingConfig.RampingVersionPercentage = 0
|
|
pendingRoutingConfig.RampingVersionChangedTime = updateTime
|
|
pendingRoutingConfig.RampingVersionPercentageChangedTime = updateTime
|
|
}
|
|
|
|
// TODO (Shivam): remove the empty string check once canary stops flaking out
|
|
if newCurrentVersion != worker_versioning.UnversionedVersionId && newCurrentVersion != "" {
|
|
// Tell new current version that it's current
|
|
currUpdateArgs := &deploymentspb.SyncVersionStateUpdateArgs{
|
|
RoutingUpdateTime: updateTime,
|
|
CurrentSinceTime: updateTime,
|
|
RampingSinceTime: nil, // remove ramp for that version if it was ramping
|
|
RampPercentage: 0, // remove ramp for that version if it was ramping
|
|
RoutingConfig: routingConfigToSync,
|
|
}
|
|
if _, err := d.syncVersion(ctx, newCurrentVersion, currUpdateArgs); err != nil {
|
|
return nil, err
|
|
}
|
|
// Erase summary drainage status immediately (in case it was previously drained/draining)
|
|
d.setDrainageStatus(newCurrentVersion, enumspb.VERSION_DRAINAGE_STATUS_UNSPECIFIED, updateTime)
|
|
}
|
|
// If the new current version is unversioned and there was no unversioned ramp, all we need to
|
|
// do is tell the previous current version that it is not current. Then, the task queues in the
|
|
// previous current version will have no current version and will become unversioned implicitly.
|
|
|
|
commitRoutingFirst := workflow.GetVersion(ctx, "commit-routing-first", workflow.DefaultVersion, 0) >= 0
|
|
if asyncMode && commitRoutingFirst {
|
|
// Commit routing config before demoting old version to avoid partial state on step 2 failure.
|
|
d.State.RoutingConfig = pendingRoutingConfig
|
|
|
|
// Fire-and-forget signal to prev version (replaces blocking sync activity)
|
|
// TODO (Shivam): remove the empty string check once canary stops flaking out
|
|
if prevCurrentVersion != worker_versioning.UnversionedVersionId && prevCurrentVersion != "" {
|
|
d.signalDemoteVersion(ctx, prevCurrentVersion, pendingRoutingConfig)
|
|
d.setVersionSummaryDraining(prevCurrentVersion, updateTime)
|
|
}
|
|
} else {
|
|
// TODO (Shivam): remove the empty string check once canary stops flaking out
|
|
if prevCurrentVersion != worker_versioning.UnversionedVersionId && prevCurrentVersion != "" {
|
|
// Tell previous current that it's no longer current
|
|
prevUpdateArgs := &deploymentspb.SyncVersionStateUpdateArgs{
|
|
RoutingUpdateTime: updateTime,
|
|
CurrentSinceTime: nil, // remove current
|
|
RampingSinceTime: nil, // no change, the prev current was not ramping
|
|
RampPercentage: 0, // no change, the prev current was not ramping
|
|
RoutingConfig: routingConfigToSync,
|
|
}
|
|
if _, err := d.syncVersion(ctx, prevCurrentVersion, prevUpdateArgs); err != nil {
|
|
return nil, err
|
|
}
|
|
if !asyncMode {
|
|
// Set summary drainage status immediately to draining.
|
|
// We know prevCurrentVersion cannot have been ramping, so it must now be draining
|
|
d.setDrainageStatus(prevCurrentVersion, enumspb.VERSION_DRAINAGE_STATUS_DRAINING, updateTime)
|
|
}
|
|
}
|
|
|
|
//nolint:staticcheck // deprecated stuff will be cleaned
|
|
if newCurrentVersion == worker_versioning.UnversionedVersionId && d.State.RoutingConfig.RampingVersion == worker_versioning.UnversionedVersionId &&
|
|
// this step is not needed in async mode because the task queues got full routing info (including removed ramp) through the previous version.
|
|
!asyncMode {
|
|
// If the new current is unversioned, and it was previously ramping, we need to tell
|
|
// all the task queues with unversioned ramp that they no longer have unversioned ramp.
|
|
// The task queues with unversioned ramp are the task queues of the previous current version.
|
|
// TODO (Carly): Should we ban people from changing the task queues in the current version while they have an unversioned ramp?
|
|
unsetRampUpdateArgs := &deploymentspb.SyncVersionStateUpdateArgs{
|
|
RoutingUpdateTime: updateTime,
|
|
RampingSinceTime: nil, // remove ramp
|
|
RampPercentage: 0, // remove ramp
|
|
}
|
|
if err := d.syncUnversionedRamp(ctx, unsetRampUpdateArgs); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// If the previous current version was unversioned, there is nothing in the task queues
|
|
// to remove, because they were implicitly unversioned. We don't have to remove any
|
|
// unversioned ramps, because current and ramping cannot both be unversioned.
|
|
|
|
d.State.RoutingConfig = pendingRoutingConfig
|
|
}
|
|
}
|
|
|
|
d.State.ConflictToken, _ = updateTime.AsTime().MarshalBinary()
|
|
d.State.LastModifierIdentity = args.Identity
|
|
|
|
// update memo
|
|
if err = d.updateMemo(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &deploymentspb.SetCurrentVersionResponse{
|
|
PreviousVersion: prevCurrentVersion,
|
|
ConflictToken: d.State.ConflictToken,
|
|
}, nil
|
|
}
|
|
|
|
// to-be-deprecated
|
|
func (d *WorkflowRunner) validateAddVersionToWorkerDeployment(args *deploymentspb.AddVersionUpdateArgs) error {
|
|
if d.State.Versions == nil {
|
|
return nil
|
|
}
|
|
|
|
for _, v := range d.State.Versions {
|
|
if v.Version == args.Version {
|
|
return temporal.NewApplicationError("deployment version already registered", errVersionAlreadyExistsType)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *WorkflowRunner) getMaxVersions(ctx workflow.Context) int {
|
|
getMaxVersionsInDeployment := func(ctx workflow.Context) any {
|
|
return d.unsafeMaxVersion()
|
|
}
|
|
intEq := func(a, b any) bool {
|
|
return a == b
|
|
}
|
|
var maxVersions int
|
|
if err := workflow.MutableSideEffect(ctx, "getMaxVersions", getMaxVersionsInDeployment, intEq).Get(&maxVersions); err != nil {
|
|
// This should not happen really. but just in case.
|
|
return defaultMaxVersions
|
|
}
|
|
return maxVersions
|
|
}
|
|
|
|
func (d *WorkflowRunner) tryDeleteVersion(ctx workflow.Context) error {
|
|
sortedSummaries := d.sortedSummaries()
|
|
for _, v := range sortedSummaries {
|
|
args := &deploymentspb.DeleteVersionArgs{
|
|
Identity: serverDeleteVersionIdentity,
|
|
Version: v.Version,
|
|
ServerDelete: true,
|
|
}
|
|
if err := d.validateDeleteVersion(args); err == nil {
|
|
// this might hang on the lock
|
|
if err = d.deleteVersion(ctx, args); err == nil {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
return serviceerror.NewFailedPrecondition("could not add version: too many versions in deployment and none are eligible for deletion")
|
|
}
|
|
|
|
func (d *WorkflowRunner) syncVersion(ctx workflow.Context, targetVersion string, versionUpdateArgs *deploymentspb.SyncVersionStateUpdateArgs) (*deploymentspb.VersionLocalState, error) {
|
|
bld := worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(targetVersion).GetBuildId()
|
|
revisionNumber := versionUpdateArgs.GetRoutingConfig().GetRevisionNumber()
|
|
if revisionNumber > 0 {
|
|
// track revision number until propagation finishes
|
|
if len(d.State.PropagatingRevisions) == 0 {
|
|
d.State.PropagatingRevisions = make(map[string]*deploymentspb.PropagatingRevisions)
|
|
}
|
|
revs, ok := d.State.PropagatingRevisions[bld]
|
|
if !ok {
|
|
revs = &deploymentspb.PropagatingRevisions{}
|
|
d.State.PropagatingRevisions[bld] = revs
|
|
}
|
|
revs.RevisionNumbers = append(revs.RevisionNumbers, revisionNumber)
|
|
}
|
|
|
|
var reqID string
|
|
if d.hasMinVersion(VersionDataRevisionNumber) && revisionNumber > 0 {
|
|
// Don't send repetitive sync requests for the same revision number to the same version.
|
|
reqID = fmt.Sprintf("SyncWorkerDeploymentVersion-%d", revisionNumber)
|
|
} else {
|
|
reqID = d.newUUID(ctx)
|
|
}
|
|
activityCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
var res deploymentspb.SyncVersionStateActivityResult
|
|
err := workflow.ExecuteActivity(activityCtx, d.a.SyncWorkerDeploymentVersion, &deploymentspb.SyncVersionStateActivityArgs{
|
|
DeploymentName: d.DeploymentName,
|
|
Version: targetVersion,
|
|
UpdateArgs: versionUpdateArgs,
|
|
RequestId: reqID,
|
|
}).Get(ctx, &res)
|
|
|
|
if err == nil {
|
|
if sum := res.GetSummary(); sum != nil {
|
|
d.updateVersionSummary(sum)
|
|
} else {
|
|
//nolint:staticcheck // SA1019
|
|
d.updateVersionSummary(versionStateToSummary(res.GetVersionState()))
|
|
}
|
|
} else if revisionNumber > 0 {
|
|
// Activity failed meaning the synchronous part of the sync request failed. we need to untrack the revision number.
|
|
d.handlePropagationComplete(&deploymentspb.PropagationCompletionInfo{RevisionNumber: revisionNumber, BuildId: bld})
|
|
}
|
|
return res.VersionState, err
|
|
}
|
|
|
|
// signalDemoteVersion sends a signal to a version workflow to demote it with the given
|
|
// routing config. The version workflow will determine its new status and propagate to task
|
|
// queues.
|
|
func (d *WorkflowRunner) signalDemoteVersion(ctx workflow.Context, version string, routingConfig *deploymentpb.RoutingConfig) {
|
|
v := worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(version)
|
|
bld := v.GetBuildId()
|
|
revisionNumber := routingConfig.GetRevisionNumber()
|
|
if revisionNumber > 0 {
|
|
// Track revision number until the version workflow signals PropagationComplete.
|
|
// Without this, RoutingConfigUpdateState never transitions to COMPLETED.
|
|
if len(d.State.PropagatingRevisions) == 0 {
|
|
d.State.PropagatingRevisions = make(map[string]*deploymentspb.PropagatingRevisions)
|
|
}
|
|
revs, ok := d.State.PropagatingRevisions[bld]
|
|
if !ok {
|
|
revs = &deploymentspb.PropagatingRevisions{}
|
|
d.State.PropagatingRevisions[bld] = revs
|
|
}
|
|
revs.RevisionNumbers = append(revs.RevisionNumbers, revisionNumber)
|
|
}
|
|
|
|
wfID := GenerateVersionWorkflowID(v.GetDeploymentName(), bld)
|
|
err := workflow.SignalExternalWorkflow(ctx, wfID, "", DemoteVersionSignalName, &deploymentspb.DemoteVersionSignalArgs{
|
|
RoutingConfig: routingConfig,
|
|
}).Get(ctx, nil)
|
|
if err != nil {
|
|
d.logger.Error("failed to signal demote version", "version", version, "error", err)
|
|
// Signal delivery failed — untrack the revision so we don't block forever.
|
|
if revisionNumber > 0 {
|
|
d.handlePropagationComplete(&deploymentspb.PropagationCompletionInfo{RevisionNumber: revisionNumber, BuildId: bld})
|
|
}
|
|
}
|
|
}
|
|
|
|
// syncUnversionedRamp should not be called in async mode
|
|
func (d *WorkflowRunner) syncUnversionedRamp(ctx workflow.Context, versionUpdateArgs *deploymentspb.SyncVersionStateUpdateArgs) error {
|
|
activityCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
|
|
// DescribeVersion activity to get all the task queues in the current version, or the ramping version if current is nil
|
|
version := d.State.RoutingConfig.CurrentVersion //nolint:staticcheck // SA1019: worker versioning v0.31
|
|
if version == worker_versioning.UnversionedVersionId {
|
|
version = d.State.RoutingConfig.RampingVersion //nolint:staticcheck // SA1019: worker versioning v0.31
|
|
}
|
|
|
|
if d.rampingVersionStringUnversioned(version) {
|
|
return nil
|
|
}
|
|
|
|
var res deploymentspb.DescribeVersionFromWorkerDeploymentActivityResult
|
|
err := workflow.ExecuteActivity(
|
|
activityCtx,
|
|
d.a.DescribeVersionFromWorkerDeployment,
|
|
&deploymentspb.DescribeVersionFromWorkerDeploymentActivityArgs{
|
|
Version: version,
|
|
}).Get(ctx, &res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// send in the task-queue families in batches of syncBatchSize
|
|
batches := make([][]*deploymentspb.SyncDeploymentVersionUserDataRequest_SyncUserData, 0)
|
|
syncReqs := make([]*deploymentspb.SyncDeploymentVersionUserDataRequest_SyncUserData, 0)
|
|
|
|
// Grouping by task-queue name
|
|
taskQueuesByName := make(map[string][]enumspb.TaskQueueType)
|
|
for _, tq := range res.GetTaskQueueInfos() {
|
|
taskQueuesByName[tq.GetName()] = append(taskQueuesByName[tq.GetName()], tq.GetType())
|
|
}
|
|
|
|
for _, tqName := range workflow.DeterministicKeys(taskQueuesByName) {
|
|
tqTypes := taskQueuesByName[tqName]
|
|
sync := &deploymentspb.SyncDeploymentVersionUserDataRequest_SyncUserData{
|
|
Name: tqName,
|
|
Types: tqTypes,
|
|
Data: &deploymentspb.DeploymentVersionData{
|
|
Version: nil,
|
|
RoutingUpdateTime: versionUpdateArgs.RoutingUpdateTime,
|
|
RampingSinceTime: versionUpdateArgs.RampingSinceTime,
|
|
RampPercentage: versionUpdateArgs.RampPercentage,
|
|
},
|
|
}
|
|
syncReqs = append(syncReqs, sync)
|
|
|
|
if len(syncReqs) == int(d.State.SyncBatchSize) {
|
|
batches = append(batches, syncReqs)
|
|
syncReqs = make([]*deploymentspb.SyncDeploymentVersionUserDataRequest_SyncUserData, 0) // reset the syncReq.Sync slice for the next batch
|
|
}
|
|
}
|
|
if len(syncReqs) > 0 {
|
|
batches = append(batches, syncReqs)
|
|
}
|
|
|
|
// calling SyncDeploymentVersionUserData for each batch
|
|
for _, batch := range batches {
|
|
var syncRes deploymentspb.SyncDeploymentVersionUserDataResponse
|
|
|
|
err = workflow.ExecuteActivity(activityCtx, d.a.SyncDeploymentVersionUserDataFromWorkerDeployment, &deploymentspb.SyncDeploymentVersionUserDataRequest{
|
|
DeploymentName: d.DeploymentName,
|
|
Version: nil,
|
|
ForgetVersion: false,
|
|
Sync: batch,
|
|
}).Get(ctx, &syncRes)
|
|
if err != nil {
|
|
// TODO (Shivam): Compensation functions required to roll back the local state + activity changes.
|
|
return err
|
|
}
|
|
|
|
if len(syncRes.TaskQueueMaxVersions) > 0 {
|
|
// wait for propagation
|
|
err = workflow.ExecuteActivity(
|
|
activityCtx,
|
|
d.a.CheckUnversionedRampUserDataPropagation,
|
|
&deploymentspb.CheckWorkerDeploymentUserDataPropagationRequest{
|
|
TaskQueueMaxVersions: syncRes.TaskQueueMaxVersions,
|
|
}).Get(ctx, nil)
|
|
if err != nil {
|
|
// TODO (Shivam): Compensation functions required to roll back the local state + activity changes.
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func (d *WorkflowRunner) isVersionMissingTaskQueues(ctx workflow.Context, prevCurrentVersion string, newCurrentVersion string) (bool, error) {
|
|
activityCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
var res deploymentspb.IsVersionMissingTaskQueuesResult
|
|
err := workflow.ExecuteActivity(activityCtx, d.a.IsVersionMissingTaskQueues, &deploymentspb.IsVersionMissingTaskQueuesArgs{
|
|
PrevCurrentVersion: prevCurrentVersion,
|
|
NewCurrentVersion: newCurrentVersion,
|
|
}).Get(ctx, &res)
|
|
return res.IsMissingTaskQueues, err
|
|
}
|
|
|
|
func (d *WorkflowRunner) startVersion(ctx workflow.Context, args *deploymentspb.StartWorkerDeploymentVersionRequest) error {
|
|
activityCtx := workflow.WithActivityOptions(ctx, defaultActivityOptions)
|
|
return workflow.ExecuteActivity(activityCtx, d.a.StartWorkerDeploymentVersionWorkflow, args).Get(ctx, nil)
|
|
}
|
|
|
|
func (d *WorkflowRunner) newUUID(ctx workflow.Context) string {
|
|
var val string
|
|
_ = workflow.SideEffect(ctx, func(ctx workflow.Context) any {
|
|
return uuid.NewString()
|
|
}).Get(&val)
|
|
return val
|
|
}
|
|
|
|
func (d *WorkflowRunner) updateMemo(ctx workflow.Context) error {
|
|
// TODO(carlydf): remove verbose logging
|
|
d.logger.Info("Updating workflow memo",
|
|
"routing_config", d.State.GetRoutingConfig(),
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
"current_version", d.State.GetRoutingConfig().GetCurrentVersion(),
|
|
//nolint:staticcheck // SA1019: worker versioning v0.31
|
|
"ramping_version", d.State.GetRoutingConfig().GetRampingVersion())
|
|
|
|
return workflow.UpsertMemo(ctx, map[string]any{
|
|
WorkerDeploymentMemoField: &deploymentspb.WorkerDeploymentWorkflowMemo{
|
|
DeploymentName: d.DeploymentName,
|
|
CreateTime: d.State.CreateTime,
|
|
RoutingConfig: d.State.RoutingConfig,
|
|
LatestVersionSummary: d.getLatestVersionSummary(),
|
|
CurrentVersionSummary: d.getCurrentVersionSummary(),
|
|
RampingVersionSummary: d.getRampingVersionSummary(),
|
|
},
|
|
})
|
|
}
|
|
|
|
func (d *WorkflowRunner) setStateChanged() {
|
|
d.stateChanged = true
|
|
}
|
|
|
|
func (d *WorkflowRunner) sortedSummaries() []*deploymentspb.WorkerDeploymentVersionSummary {
|
|
var sortedSummaries []*deploymentspb.WorkerDeploymentVersionSummary
|
|
for _, k := range workflow.DeterministicKeys(d.State.Versions) {
|
|
s := d.State.Versions[k]
|
|
sortedSummaries = append(sortedSummaries, s)
|
|
}
|
|
|
|
slices.SortFunc(sortedSummaries, func(a, b *deploymentspb.WorkerDeploymentVersionSummary) int {
|
|
// sorts in ascending order.
|
|
// cmp(a, b) should return a negative number when a < b, a positive number when a > b,
|
|
// and zero when a == b or a and b are incomparable in the sense of a strict weak ordering.
|
|
if a.GetCreateTime().AsTime().After(b.GetCreateTime().AsTime()) {
|
|
return 1
|
|
} else if a.GetCreateTime().AsTime().Before(b.GetCreateTime().AsTime()) {
|
|
return -1
|
|
}
|
|
return 0
|
|
})
|
|
return sortedSummaries
|
|
}
|
|
|
|
func (d *WorkflowRunner) getLatestVersionSummary() *deploymentpb.WorkerDeploymentInfo_WorkerDeploymentVersionSummary {
|
|
sortedSummaries := d.sortedSummaries()
|
|
if len(sortedSummaries) == 0 {
|
|
return nil
|
|
}
|
|
latest_summary := sortedSummaries[len(sortedSummaries)-1]
|
|
return d.getWorkerDeploymentInfoVersionSummary(latest_summary)
|
|
}
|
|
|
|
func (d *WorkflowRunner) getCurrentVersionSummary() *deploymentpb.WorkerDeploymentInfo_WorkerDeploymentVersionSummary {
|
|
// The deployment workflow still uses the deprecated fields from v0.31. Hence, the current version is read from
|
|
// CurrentVersion and not CurrentDeploymentVersion. This shall change before GA.
|
|
currentVersion := d.GetState().GetRoutingConfig().GetCurrentVersion() //nolint:staticcheck
|
|
currentVersionSummary := d.GetState().GetVersions()[currentVersion]
|
|
|
|
if currentVersionSummary == nil {
|
|
return nil
|
|
}
|
|
return d.getWorkerDeploymentInfoVersionSummary(currentVersionSummary)
|
|
}
|
|
|
|
func (d *WorkflowRunner) getRampingVersionSummary() *deploymentpb.WorkerDeploymentInfo_WorkerDeploymentVersionSummary {
|
|
// The deployment workflow still uses the deprecated fields from v0.31. Hence, the ramping version is read from
|
|
// RampingVersion and not RampingDeploymentVersion. This shall change before GA.
|
|
rampingVersion := d.GetState().GetRoutingConfig().GetRampingVersion() //nolint:staticcheck
|
|
rampingVersionSummary := d.GetState().GetVersions()[rampingVersion]
|
|
|
|
if rampingVersionSummary == nil {
|
|
return nil
|
|
}
|
|
return d.getWorkerDeploymentInfoVersionSummary(rampingVersionSummary)
|
|
}
|
|
|
|
func (d *WorkflowRunner) getWorkerDeploymentInfoVersionSummary(versionSummary *deploymentspb.WorkerDeploymentVersionSummary) *deploymentpb.WorkerDeploymentInfo_WorkerDeploymentVersionSummary {
|
|
return &deploymentpb.WorkerDeploymentInfo_WorkerDeploymentVersionSummary{
|
|
Version: versionSummary.GetVersion(),
|
|
DeploymentVersion: worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(versionSummary.GetVersion()),
|
|
Status: versionSummary.GetStatus(),
|
|
CreateTime: versionSummary.GetCreateTime(),
|
|
DrainageInfo: versionSummary.GetDrainageInfo(),
|
|
CurrentSinceTime: versionSummary.GetCurrentSinceTime(),
|
|
RampingSinceTime: versionSummary.GetRampingSinceTime(),
|
|
RoutingUpdateTime: versionSummary.GetRoutingUpdateTime(),
|
|
FirstActivationTime: versionSummary.GetFirstActivationTime(),
|
|
LastCurrentTime: versionSummary.GetLastCurrentTime(),
|
|
LastDeactivationTime: versionSummary.GetLastDeactivationTime(),
|
|
ComputeConfig: versionSummary.GetComputeConfig(),
|
|
}
|
|
}
|