mirror of
https://github.com/temporalio/temporal.git
synced 2026-08-30 18:41:49 -07:00
Resend child workflow async when missing on passive (#11705)
## What changed? - Added an opt-in recovery path for a child workflow missing from a standby cluster. - After the existing resend delay, VerifyFirstWorkflowTaskScheduled asynchronously fetches the child state from the active cluster, applies it locally, and verifies it again. - Added deduplication, per-shard concurrency limits, metrics, namespace checks, and transition-history gating. Corrected the discard-time source check to verify the child workflow rather than the parent. - Moved the reusable in-flight resend tracker into the shared workflowresend package. - Updated the existing XDC parent-child test to assert that the missing child and its first workflow task are restored. ## Why? Cross-shard replication may deliver the parent’s ChildWorkflowExecutionStarted event before the child workflow reaches the standby cluster. Previously, verification repeatedly returned NotFound and eventually discarded the standby task, leaving the child missing. This adds the child-side symmetric recovery behavior to the parent resend implemented in #11424 . ## How did you test it? - [ ] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) ### Rollout `history.enableChildWorkflowResend` default to `false`. ### Known - Enabling this feature introduces additional cross-cluster state-sync traffic. It is disabled by default and protected by deduplication and a per-shard concurrency limit. - Regular replication may race with state sync; duplicate application is treated as success. - The resend delay and replication timeout should remain below the standby task discard delay so recovery has time to complete.
This commit is contained in:
@@ -3718,6 +3718,7 @@ type VerifyFirstWorkflowTaskScheduledRequest struct {
|
||||
NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"`
|
||||
WorkflowExecution *v14.WorkflowExecution `protobuf:"bytes,2,opt,name=workflow_execution,json=workflowExecution,proto3" json:"workflow_execution,omitempty"`
|
||||
Clock *v18.VectorClock `protobuf:"bytes,3,opt,name=clock,proto3" json:"clock,omitempty"`
|
||||
ResendChild bool `protobuf:"varint,4,opt,name=resend_child,json=resendChild,proto3" json:"resend_child,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -3773,6 +3774,13 @@ func (x *VerifyFirstWorkflowTaskScheduledRequest) GetClock() *v18.VectorClock {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *VerifyFirstWorkflowTaskScheduledRequest) GetResendChild() bool {
|
||||
if x != nil {
|
||||
return x.ResendChild
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type VerifyFirstWorkflowTaskScheduledResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
@@ -11018,11 +11026,12 @@ const file_temporal_server_api_historyservice_v1_request_response_proto_rawDesc
|
||||
"\vchild_clock\x18\x04 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\n" +
|
||||
"childClock\x12L\n" +
|
||||
"\fparent_clock\x18\x05 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\vparentClock:$\x92\xc4\x03 *\x1eworkflow_execution.workflow_id\"\x1e\n" +
|
||||
"\x1cScheduleWorkflowTaskResponse\"\x8d\x02\n" +
|
||||
"\x1cScheduleWorkflowTaskResponse\"\xb0\x02\n" +
|
||||
"'VerifyFirstWorkflowTaskScheduledRequest\x12!\n" +
|
||||
"\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12X\n" +
|
||||
"\x12workflow_execution\x18\x02 \x01(\v2).temporal.api.common.v1.WorkflowExecutionR\x11workflowExecution\x12?\n" +
|
||||
"\x05clock\x18\x03 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\x05clock:$\x92\xc4\x03 *\x1eworkflow_execution.workflow_id\"*\n" +
|
||||
"\x05clock\x18\x03 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\x05clock\x12!\n" +
|
||||
"\fresend_child\x18\x04 \x01(\bR\vresendChild:$\x92\xc4\x03 *\x1eworkflow_execution.workflow_id\"*\n" +
|
||||
"(VerifyFirstWorkflowTaskScheduledResponse\"\xd4\x04\n" +
|
||||
"$RecordChildExecutionCompletedRequest\x12!\n" +
|
||||
"\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12T\n" +
|
||||
|
||||
@@ -2796,15 +2796,8 @@ the number of children greater than or equal to this threshold`,
|
||||
ReplicationTaskApplyTimeout = NewGlobalDurationSetting(
|
||||
"history.ReplicationTaskApplyTimeout",
|
||||
20*time.Second,
|
||||
`ReplicationTaskApplyTimeout is the context timeout for replication task apply, and for the
|
||||
standby CloseExecutionTask's child-to-parent completion verification`,
|
||||
)
|
||||
ParentWorkflowResendMaxInFlight = NewGlobalIntSetting(
|
||||
"history.parentWorkflowResendMaxInFlight",
|
||||
8,
|
||||
`ParentWorkflowResendMaxInFlight caps how many parent workflow resends a shard may run
|
||||
concurrently when EnableAsyncParentWorkflowResend is on. Attempts beyond the cap are dropped; the
|
||||
verifying task retries. This bounds the goroutines this path can create per shard.`,
|
||||
`ReplicationTaskApplyTimeout is the context timeout for replication task apply, and for
|
||||
standby parent-child verification resends`,
|
||||
)
|
||||
EnableAsyncParentWorkflowResend = NewGlobalBoolSetting(
|
||||
"history.enableAsyncParentWorkflowResend",
|
||||
@@ -2812,6 +2805,21 @@ verifying task retries. This bounds the goroutines this path can create per shar
|
||||
`EnableAsyncParentWorkflowResend controls whether the standby child-to-parent completion
|
||||
verification resends the parent workflow in the background rather than inline, so the verifying task
|
||||
is not held for the duration of the cross-cluster sync.`,
|
||||
)
|
||||
EnableChildWorkflowResend = NewGlobalBoolSetting(
|
||||
"history.enableChildWorkflowResend",
|
||||
false,
|
||||
`EnableChildWorkflowResend controls whether standby parent-to-child first workflow task
|
||||
verification may resend a missing child workflow in the background from the active cluster. When
|
||||
disabled, verification remains local-only. StandbyTaskMissingEventsResendDelay plus
|
||||
ReplicationTaskApplyTimeout should remain below StandbyTaskMissingEventsDiscardDelay.`,
|
||||
)
|
||||
WorkflowResendHostMaxInFlight = NewGlobalIntSetting(
|
||||
"history.workflowResendHostMaxInFlight",
|
||||
16,
|
||||
`WorkflowResendHostMaxInFlight caps the total number of asynchronous parent and child workflow
|
||||
resends that may run concurrently on a history host. Values less than one reject all asynchronous
|
||||
workflow resends.`,
|
||||
)
|
||||
ReplicationTaskFetcherParallelism = NewGlobalIntSetting(
|
||||
"history.ReplicationTaskFetcherParallelism",
|
||||
|
||||
@@ -1137,12 +1137,24 @@ var (
|
||||
ParentWorkflowResendAttempts = NewCounterDef("parent_workflow_resend_attempts")
|
||||
// ParentWorkflowResendSkipped counts attempts that found a resend for the same parent in flight.
|
||||
ParentWorkflowResendSkipped = NewCounterDef("parent_workflow_resend_skipped")
|
||||
// ParentWorkflowResendFailures counts failed resends. Async resends report failure nowhere else.
|
||||
// ParentWorkflowResendFailures counts unexpected parent resend failures.
|
||||
ParentWorkflowResendFailures = NewCounterDef("parent_workflow_resend_failures")
|
||||
// ParentWorkflowResendLimited counts resends dropped because the shard was at its in-flight cap.
|
||||
// ParentWorkflowResendLimited counts parent resends rejected at the host-level concurrency limit.
|
||||
ParentWorkflowResendLimited = NewCounterDef("parent_workflow_resend_limited")
|
||||
// ParentWorkflowResendLatency measures a resend: cross-cluster state fetch plus local apply.
|
||||
// ParentWorkflowResendLatency measures a parent resend and subsequent verification.
|
||||
ParentWorkflowResendLatency = NewTimerDef("parent_workflow_resend_latency")
|
||||
// ChildWorkflowResendAttempts counts child resends started by standby first-task verification.
|
||||
ChildWorkflowResendAttempts = NewCounterDef("child_workflow_resend_attempts")
|
||||
// ChildWorkflowResendSkipped counts attempts that found a resend for the same child in flight.
|
||||
ChildWorkflowResendSkipped = NewCounterDef("child_workflow_resend_skipped")
|
||||
// ChildWorkflowResendFailures counts unexpected child resend failures.
|
||||
ChildWorkflowResendFailures = NewCounterDef("child_workflow_resend_failures")
|
||||
// ChildWorkflowResendLimited counts child resends rejected at the host-level concurrency limit.
|
||||
ChildWorkflowResendLimited = NewCounterDef("child_workflow_resend_limited")
|
||||
// ChildWorkflowResendLatency measures a child resend and subsequent verification.
|
||||
ChildWorkflowResendLatency = NewTimerDef("child_workflow_resend_latency")
|
||||
// WorkflowResendSchedulerAtCapacity counts host-level workflow resends rejected at the concurrency limit.
|
||||
WorkflowResendSchedulerAtCapacity = NewCounterDef("workflow_resend_scheduler_at_capacity")
|
||||
// ReplicationOrphanedHistoryBranch tracks cases where history branch cleanup was skipped on error
|
||||
// to avoid deleting successfully written history. These orphaned branches will be cleaned up by GC.
|
||||
ReplicationOrphanedHistoryBranch = NewCounterDef("replication_orphaned_history_branch")
|
||||
|
||||
@@ -593,6 +593,7 @@ message VerifyFirstWorkflowTaskScheduledRequest {
|
||||
string namespace_id = 1;
|
||||
temporal.api.common.v1.WorkflowExecution workflow_execution = 2;
|
||||
temporal.server.api.clock.v1.VectorClock clock = 3;
|
||||
bool resend_child = 4;
|
||||
}
|
||||
|
||||
message VerifyFirstWorkflowTaskScheduledResponse {}
|
||||
|
||||
@@ -3,21 +3,18 @@ package verifychildworkflowcompletionrecorded
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"time"
|
||||
|
||||
commonpb "go.temporal.io/api/common/v1"
|
||||
"go.temporal.io/api/serviceerror"
|
||||
"go.temporal.io/server/api/adminservice/v1"
|
||||
enumsspb "go.temporal.io/server/api/enums/v1"
|
||||
historyspb "go.temporal.io/server/api/history/v1"
|
||||
"go.temporal.io/server/api/historyservice/v1"
|
||||
persistencespb "go.temporal.io/server/api/persistence/v1"
|
||||
"go.temporal.io/server/chasm"
|
||||
"go.temporal.io/server/common"
|
||||
"go.temporal.io/server/common/definition"
|
||||
"go.temporal.io/server/common/locks"
|
||||
"go.temporal.io/server/common/log"
|
||||
"go.temporal.io/server/common/log/tag"
|
||||
"go.temporal.io/server/common/metrics"
|
||||
"go.temporal.io/server/common/namespace"
|
||||
@@ -27,6 +24,7 @@ import (
|
||||
"go.temporal.io/server/common/util"
|
||||
"go.temporal.io/server/common/wideevents"
|
||||
"go.temporal.io/server/service/history/api"
|
||||
"go.temporal.io/server/service/history/api/workflowresend"
|
||||
"go.temporal.io/server/service/history/consts"
|
||||
historyi "go.temporal.io/server/service/history/interfaces"
|
||||
)
|
||||
@@ -101,7 +99,7 @@ func Invoke(
|
||||
request *historyservice.VerifyChildExecutionCompletionRecordedRequest,
|
||||
workflowConsistencyChecker api.WorkflowConsistencyChecker,
|
||||
shardContext historyi.ShardContext,
|
||||
inFlightResends *InFlightResends,
|
||||
resendScheduler workflowresend.Scheduler,
|
||||
) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) {
|
||||
namespaceID := namespace.ID(request.GetNamespaceId())
|
||||
if err := api.ValidateNamespaceUUID(namespaceID); err != nil {
|
||||
@@ -122,51 +120,68 @@ func Invoke(
|
||||
|
||||
metricsHandler := shardContext.GetMetricsHandler()
|
||||
emitLifecycle := shardContext.GetConfig().EmitReplicationLifecycleEvents()
|
||||
asyncResend := shardContext.GetConfig().EnableAsyncParentWorkflowResend()
|
||||
|
||||
// The measured resend, run either inline or in the background.
|
||||
resend := func(ctx context.Context) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) {
|
||||
resend := func(ctx context.Context) error {
|
||||
metrics.ParentWorkflowResendAttempts.With(metricsHandler).Record(1)
|
||||
startTime := time.Now().UTC()
|
||||
resp, err := resendParentAndVerify(ctx, request, workflowConsistencyChecker, shardContext, namespaceID, versionedTransition, versionHistories, errVerify, parentWorkflowState, emitLifecycle)
|
||||
err := resendParentAndVerify(
|
||||
ctx,
|
||||
request,
|
||||
workflowConsistencyChecker,
|
||||
shardContext,
|
||||
namespaceID,
|
||||
versionedTransition,
|
||||
versionHistories,
|
||||
errVerify,
|
||||
parentWorkflowState,
|
||||
emitLifecycle,
|
||||
)
|
||||
metrics.ParentWorkflowResendLatency.With(metricsHandler).Record(time.Since(startTime))
|
||||
if err != nil {
|
||||
recordResendFailure(shardContext, metricsHandler, request, err)
|
||||
if err != nil && !isExpectedResendError(err) {
|
||||
metrics.ParentWorkflowResendFailures.With(metricsHandler).Record(1)
|
||||
}
|
||||
return resp, err
|
||||
logResendFailure(shardContext, request, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if !asyncResend {
|
||||
return resend(ctx)
|
||||
if resendScheduler == nil || !shardContext.GetConfig().EnableAsyncParentWorkflowResend() {
|
||||
if err := resend(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil
|
||||
}
|
||||
|
||||
// The resend can take minutes while the calling standby task's deadline is short, so run it in
|
||||
// the background and let that task retry until the parent lands.
|
||||
//
|
||||
// At most one resend per parent, and at most ParentWorkflowResendMaxInFlight per shard: callers
|
||||
// retry while an earlier resend runs, so without these a stale parent, or a namespace with many
|
||||
// of them, would spawn goroutines without bound.
|
||||
// The host scheduler deduplicates by workflow and applies an aggregate limit across shards.
|
||||
parentKey := definition.NewWorkflowKey(request.NamespaceId, request.ParentExecution.WorkflowId, request.ParentExecution.RunId)
|
||||
maxInFlight := shardContext.GetConfig().ParentWorkflowResendMaxInFlight()
|
||||
claimed, atCapacity := inFlightResends.tryClaim(parentKey, maxInFlight)
|
||||
if atCapacity {
|
||||
metrics.ParentWorkflowResendLimited.With(metricsHandler).Record(1)
|
||||
if emitLifecycle {
|
||||
details := parentResendEventDetails(errVerify)
|
||||
details["max_in_flight"] = maxInFlight
|
||||
emitParentResendLifecycleEvent(shardContext, request, parentWorkflowState, wideevents.ParentChildOutcomeLimited, nil, details)
|
||||
}
|
||||
shardContext.GetLogger().Warn("Dropped parent workflow resend, shard is at its in-flight limit",
|
||||
tag.WorkflowNamespaceID(request.GetNamespaceId()),
|
||||
tag.NewStringTag("parent-workflow-id", request.ParentExecution.GetWorkflowId()),
|
||||
tag.NewStringTag("parent-run-id", request.ParentExecution.GetRunId()),
|
||||
tag.NewStringTag("child-workflow-id", request.ChildExecution.GetWorkflowId()),
|
||||
tag.NewStringTag("child-run-id", request.ChildExecution.GetRunId()),
|
||||
tag.NewInt("max-in-flight", shardContext.GetConfig().ParentWorkflowResendMaxInFlight()),
|
||||
)
|
||||
return nil, errVerify
|
||||
}
|
||||
if !claimed {
|
||||
|
||||
// The context is detached from the request, which gRPC cancels when this handler returns, and
|
||||
// rooted at the shard lifecycle so the work stops with the shard.
|
||||
resendCtx := rpc.CopyContextValues(shardContext.GetLifecycleContext(), ctx)
|
||||
submitResult := resendScheduler.TrySubmit(
|
||||
resendCtx,
|
||||
parentKey,
|
||||
shardContext.GetConfig().ReplicationTaskApplyTimeout(),
|
||||
func(ctx context.Context) {
|
||||
if emitLifecycle {
|
||||
emitParentResendLifecycleEvent(
|
||||
shardContext,
|
||||
request,
|
||||
parentWorkflowState,
|
||||
wideevents.ParentChildOutcomeScheduled,
|
||||
nil,
|
||||
parentResendEventDetails(errVerify),
|
||||
)
|
||||
}
|
||||
_ = resend(ctx)
|
||||
},
|
||||
)
|
||||
switch submitResult {
|
||||
case workflowresend.SubmitResultAccepted:
|
||||
// Accepted work records its attempt when execution starts.
|
||||
case workflowresend.SubmitResultDuplicate:
|
||||
metrics.ParentWorkflowResendSkipped.With(metricsHandler).Record(1)
|
||||
if emitLifecycle {
|
||||
emitParentResendLifecycleEvent(
|
||||
@@ -178,48 +193,38 @@ func Invoke(
|
||||
parentResendEventDetails(errVerify),
|
||||
)
|
||||
}
|
||||
return nil, errVerify
|
||||
case workflowresend.SubmitResultAtCapacity:
|
||||
metrics.ParentWorkflowResendLimited.With(metricsHandler).Record(1)
|
||||
if emitLifecycle {
|
||||
details := parentResendEventDetails(errVerify)
|
||||
details["max_in_flight"] = shardContext.GetConfig().WorkflowResendHostMaxInFlight()
|
||||
emitParentResendLifecycleEvent(
|
||||
shardContext,
|
||||
request,
|
||||
parentWorkflowState,
|
||||
wideevents.ParentChildOutcomeLimited,
|
||||
nil,
|
||||
details,
|
||||
)
|
||||
}
|
||||
default:
|
||||
metrics.ParentWorkflowResendFailures.With(metricsHandler).Record(1)
|
||||
}
|
||||
if emitLifecycle {
|
||||
emitParentResendLifecycleEvent(
|
||||
shardContext,
|
||||
request,
|
||||
parentWorkflowState,
|
||||
wideevents.ParentChildOutcomeScheduled,
|
||||
nil,
|
||||
parentResendEventDetails(errVerify),
|
||||
)
|
||||
}
|
||||
|
||||
// The context is detached from the request, which gRPC cancels when this handler returns, and
|
||||
// rooted at the shard lifecycle so the work stops with the shard.
|
||||
resendCtx := rpc.CopyContextValues(shardContext.GetLifecycleContext(), ctx)
|
||||
resendCtx, cancel := context.WithTimeout(resendCtx, shardContext.GetConfig().ReplicationTaskApplyTimeout())
|
||||
go func() {
|
||||
defer cancel()
|
||||
defer inFlightResends.release(parentKey)
|
||||
defer func() {
|
||||
var panicErr error
|
||||
log.CapturePanic(shardContext.GetLogger(), &panicErr)
|
||||
if panicErr != nil {
|
||||
metrics.ParentWorkflowResendFailures.With(metricsHandler).Record(1)
|
||||
}
|
||||
}()
|
||||
_, _ = resend(resendCtx)
|
||||
}()
|
||||
// The submission result is intentionally used only for metrics. Preserve the verification error
|
||||
// so the durable standby task retries regardless of the admission outcome.
|
||||
return nil, errVerify
|
||||
}
|
||||
|
||||
// recordResendFailure reports a failed parent resend. On the asynchronous path no caller receives
|
||||
// the error, so these are its only signals.
|
||||
func recordResendFailure(
|
||||
func logResendFailure(
|
||||
shardContext historyi.ShardContext,
|
||||
metricsHandler metrics.Handler,
|
||||
request *historyservice.VerifyChildExecutionCompletionRecordedRequest,
|
||||
err error,
|
||||
) {
|
||||
metrics.ParentWorkflowResendFailures.With(metricsHandler).Record(1)
|
||||
shardContext.GetLogger().Error("Failed to resend parent workflow for child completion verification",
|
||||
if isExpectedResendError(err) {
|
||||
return
|
||||
}
|
||||
shardContext.GetThrottledLogger().Error(
|
||||
"Failed to resend parent workflow for child completion verification",
|
||||
tag.WorkflowNamespaceID(request.GetNamespaceId()),
|
||||
tag.NewStringTag("parent-workflow-id", request.ParentExecution.GetWorkflowId()),
|
||||
tag.NewStringTag("parent-run-id", request.ParentExecution.GetRunId()),
|
||||
@@ -229,6 +234,18 @@ func recordResendFailure(
|
||||
)
|
||||
}
|
||||
|
||||
func isExpectedResendError(err error) bool {
|
||||
if err == nil || common.IsContextCanceledErr(err) {
|
||||
return true
|
||||
}
|
||||
var notFoundErr *serviceerror.NotFound
|
||||
var workflowNotReadyErr *serviceerror.WorkflowNotReady
|
||||
var namespaceNotFoundErr *serviceerror.NamespaceNotFound
|
||||
return errors.As(err, ¬FoundErr) ||
|
||||
errors.As(err, &workflowNotReadyErr) ||
|
||||
errors.As(err, &namespaceNotFoundErr)
|
||||
}
|
||||
|
||||
// resendParentAndVerify pulls the parent workflow's state from the source cluster, applies it, and
|
||||
// re-checks the child's completion. Separate from Invoke so the async path can run it detached.
|
||||
func resendParentAndVerify(
|
||||
@@ -242,11 +259,7 @@ func resendParentAndVerify(
|
||||
errVerify error,
|
||||
parentWorkflowState string,
|
||||
emitLifecycle bool,
|
||||
) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) {
|
||||
// Resend parent workflow from source cluster
|
||||
|
||||
clusterMetadata := shardContext.GetClusterMetadata()
|
||||
targetClusterInfo := clusterMetadata.GetAllClusterInfo()[clusterMetadata.GetCurrentClusterName()]
|
||||
) error {
|
||||
activeClusterName := ""
|
||||
emitResult := func(outcome string, eventErr error, stage string) {
|
||||
if !emitLifecycle {
|
||||
@@ -261,80 +274,48 @@ func resendParentAndVerify(
|
||||
}
|
||||
emitParentResendLifecycleEvent(shardContext, request, parentWorkflowState, outcome, eventErr, details)
|
||||
}
|
||||
|
||||
namespaceEntry, err := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespaceID)
|
||||
if err != nil {
|
||||
emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_namespace")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
activeClusterName = namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: request.ParentExecution.WorkflowId})
|
||||
if activeClusterName == clusterMetadata.GetCurrentClusterName() {
|
||||
err = errors.New("namespace becomes active when processing task as standby")
|
||||
emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_source_cluster")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
remoteAdminClient, err := shardContext.GetRemoteAdminClient(activeClusterName)
|
||||
if err != nil {
|
||||
emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_remote_client")
|
||||
return nil, err
|
||||
}
|
||||
emitResult(wideevents.ParentChildOutcomeStarted, nil, "sync_workflow_state")
|
||||
|
||||
resp, err := remoteAdminClient.SyncWorkflowState(ctx, &adminservice.SyncWorkflowStateRequest{
|
||||
NamespaceId: request.NamespaceId,
|
||||
Execution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: request.ParentExecution.WorkflowId,
|
||||
RunId: request.ParentExecution.RunId,
|
||||
result, err := workflowresend.SyncWorkflowStateFromSource(
|
||||
ctx,
|
||||
shardContext,
|
||||
namespaceID,
|
||||
request.ParentExecution,
|
||||
versionedTransition,
|
||||
versionHistories,
|
||||
func(sourceCluster string) {
|
||||
activeClusterName = sourceCluster
|
||||
emitResult(wideevents.ParentChildOutcomeStarted, nil, "sync_workflow_state")
|
||||
},
|
||||
ArchetypeId: chasm.WorkflowArchetypeID,
|
||||
VersionedTransition: versionedTransition,
|
||||
VersionHistories: versionHistories,
|
||||
TargetClusterId: int32(targetClusterInfo.InitialFailoverVersion),
|
||||
})
|
||||
|
||||
)
|
||||
if err != nil {
|
||||
if common.IsNotFoundError(err) {
|
||||
// parent workflow is not found on source cluster,
|
||||
// we can return empty response to indicate that verification is done
|
||||
// TODO: add parent workflow to workflowNotFoundCache
|
||||
emitResult(wideevents.ParentChildOutcomeSourceNotFound, err, "sync_workflow_state")
|
||||
return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil
|
||||
}
|
||||
if _, ok := errors.AsType[*serviceerror.FailedPrecondition](err); ok {
|
||||
// Unable to perform sync state. Transition history maybe disabled in source cluster.
|
||||
emitResult(wideevents.ParentChildOutcomeFailed, err, "sync_workflow_state")
|
||||
return nil, errVerify
|
||||
}
|
||||
emitResult(wideevents.ParentChildOutcomeFailed, err, "sync_workflow_state")
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
switch result {
|
||||
case workflowresend.SyncWorkflowStateResultSourceNotFound:
|
||||
// TODO: add parent workflow to workflowNotFoundCache
|
||||
emitResult(
|
||||
wideevents.ParentChildOutcomeSourceNotFound,
|
||||
serviceerror.NewNotFound("parent workflow not found on source cluster"),
|
||||
"sync_workflow_state",
|
||||
)
|
||||
return nil
|
||||
case workflowresend.SyncWorkflowStateResultSkipped:
|
||||
return errVerify
|
||||
case workflowresend.SyncWorkflowStateResultApplied:
|
||||
default:
|
||||
return fmt.Errorf("unknown workflow state sync result: %d", result)
|
||||
}
|
||||
|
||||
engine, err := shardContext.GetEngine(ctx)
|
||||
if err != nil {
|
||||
emitResult(wideevents.ParentChildOutcomeFailed, err, "get_engine")
|
||||
return nil, err
|
||||
}
|
||||
err = engine.ReplicateVersionedTransition(ctx, chasm.WorkflowArchetypeID, resp.VersionedTransitionArtifact, activeClusterName)
|
||||
if err != nil {
|
||||
if !errors.Is(err, consts.ErrDuplicate) {
|
||||
emitResult(wideevents.ParentChildOutcomeFailed, err, "replicate_versioned_transition")
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Verify child execution again after resending parent workflow
|
||||
_, _, observedParentWorkflowState, err := verifyChildExecution(ctx, workflowConsistencyChecker, request)
|
||||
if observedParentWorkflowState != "" {
|
||||
parentWorkflowState = observedParentWorkflowState
|
||||
}
|
||||
if err != nil {
|
||||
emitResult(wideevents.ParentChildOutcomeFailed, err, "verify_after_resend")
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
emitResult(wideevents.ParentChildOutcomeSucceeded, nil, "")
|
||||
return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func parentResendEventDetails(initialError error) map[string]any {
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
package verifychildworkflowcompletionrecorded
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"go.temporal.io/server/common/definition"
|
||||
)
|
||||
|
||||
// InFlightResends tracks the parent workflows a shard is currently resending, so concurrent
|
||||
// verification attempts for the same parent do not each pull its state from the source cluster. A
|
||||
// resend clones the parent's full mutable state, and callers retry while one is still running.
|
||||
//
|
||||
// It also caps how many resends a shard runs at once, bounding the goroutines this path creates.
|
||||
//
|
||||
// The zero value is ready to use; hold it by pointer, never copy it.
|
||||
type InFlightResends struct {
|
||||
mu sync.Mutex
|
||||
keys map[definition.WorkflowKey]struct{}
|
||||
}
|
||||
|
||||
// tryClaim reserves key for the caller. It reports claimed=false when a resend for the same parent
|
||||
// is already running, or atCapacity=true when the shard already has maxInFlight resends. A caller
|
||||
// that claims the key must release it when the resend finishes.
|
||||
func (r *InFlightResends) tryClaim(key definition.WorkflowKey, maxInFlight int) (claimed bool, atCapacity bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, ok := r.keys[key]; ok {
|
||||
return false, false
|
||||
}
|
||||
if len(r.keys) >= maxInFlight {
|
||||
return false, true
|
||||
}
|
||||
if r.keys == nil {
|
||||
r.keys = make(map[definition.WorkflowKey]struct{})
|
||||
}
|
||||
r.keys[key] = struct{}{}
|
||||
return true, false
|
||||
}
|
||||
|
||||
func (r *InFlightResends) release(key definition.WorkflowKey) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
delete(r.keys, key)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package verifychildworkflowcompletionrecorded
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.temporal.io/server/common/definition"
|
||||
)
|
||||
|
||||
func key(workflowID string) definition.WorkflowKey {
|
||||
return definition.NewWorkflowKey("ns", workflowID, "run")
|
||||
}
|
||||
|
||||
func TestInFlightResends_ZeroValueIsUsable(t *testing.T) {
|
||||
var r InFlightResends // no constructor
|
||||
claimed, atCapacity := r.tryClaim(key("a"), 8)
|
||||
require.True(t, claimed)
|
||||
require.False(t, atCapacity)
|
||||
}
|
||||
|
||||
func TestInFlightResends_DedupesSameParent(t *testing.T) {
|
||||
var r InFlightResends
|
||||
|
||||
claimed, atCapacity := r.tryClaim(key("a"), 8)
|
||||
require.True(t, claimed)
|
||||
require.False(t, atCapacity)
|
||||
|
||||
// Same parent while the first is still held.
|
||||
claimed, atCapacity = r.tryClaim(key("a"), 8)
|
||||
require.False(t, claimed)
|
||||
require.False(t, atCapacity, "a duplicate is not a capacity problem")
|
||||
|
||||
// A different parent is unaffected.
|
||||
claimed, _ = r.tryClaim(key("b"), 8)
|
||||
require.True(t, claimed)
|
||||
|
||||
// Releasing lets the parent be claimed again.
|
||||
r.release(key("a"))
|
||||
claimed, _ = r.tryClaim(key("a"), 8)
|
||||
require.True(t, claimed)
|
||||
}
|
||||
|
||||
func TestInFlightResends_EnforcesMaxInFlight(t *testing.T) {
|
||||
var r InFlightResends
|
||||
|
||||
claimed, _ := r.tryClaim(key("a"), 2)
|
||||
require.True(t, claimed)
|
||||
claimed, _ = r.tryClaim(key("b"), 2)
|
||||
require.True(t, claimed)
|
||||
|
||||
// Third distinct parent exceeds the cap.
|
||||
claimed, atCapacity := r.tryClaim(key("c"), 2)
|
||||
require.False(t, claimed)
|
||||
require.True(t, atCapacity)
|
||||
|
||||
// A duplicate of an already-held parent still reports dedup, not capacity.
|
||||
claimed, atCapacity = r.tryClaim(key("a"), 2)
|
||||
require.False(t, claimed)
|
||||
require.False(t, atCapacity)
|
||||
|
||||
// Freeing a slot admits the previously rejected parent.
|
||||
r.release(key("b"))
|
||||
claimed, atCapacity = r.tryClaim(key("c"), 2)
|
||||
require.True(t, claimed)
|
||||
require.False(t, atCapacity)
|
||||
}
|
||||
|
||||
func TestInFlightResends_ZeroMaxRejectsEverything(t *testing.T) {
|
||||
var r InFlightResends
|
||||
claimed, atCapacity := r.tryClaim(key("a"), 0)
|
||||
require.False(t, claimed)
|
||||
require.True(t, atCapacity)
|
||||
}
|
||||
@@ -2,26 +2,121 @@ package verifyfirstworkflowtaskscheduled
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.temporal.io/api/serviceerror"
|
||||
enumsspb "go.temporal.io/server/api/enums/v1"
|
||||
historyspb "go.temporal.io/server/api/history/v1"
|
||||
"go.temporal.io/server/api/historyservice/v1"
|
||||
persistencespb "go.temporal.io/server/api/persistence/v1"
|
||||
"go.temporal.io/server/common"
|
||||
"go.temporal.io/server/common/definition"
|
||||
"go.temporal.io/server/common/locks"
|
||||
"go.temporal.io/server/common/log/tag"
|
||||
"go.temporal.io/server/common/metrics"
|
||||
"go.temporal.io/server/common/namespace"
|
||||
"go.temporal.io/server/common/persistence/transitionhistory"
|
||||
"go.temporal.io/server/common/persistence/versionhistory"
|
||||
"go.temporal.io/server/common/rpc"
|
||||
"go.temporal.io/server/service/history/api"
|
||||
"go.temporal.io/server/service/history/api/workflowresend"
|
||||
"go.temporal.io/server/service/history/consts"
|
||||
historyi "go.temporal.io/server/service/history/interfaces"
|
||||
)
|
||||
|
||||
func Invoke(
|
||||
ctx context.Context,
|
||||
req *historyservice.VerifyFirstWorkflowTaskScheduledRequest,
|
||||
workflowConsistencyChecker api.WorkflowConsistencyChecker,
|
||||
) (retError error) {
|
||||
shardContext historyi.ShardContext,
|
||||
resendScheduler workflowresend.Scheduler,
|
||||
) error {
|
||||
namespaceID := namespace.ID(req.GetNamespaceId())
|
||||
if err := api.ValidateNamespaceUUID(namespaceID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versionedTransition, versionHistories, firstWorkflowTaskMissing, errVerify := verifyFirstWorkflowTaskScheduled(ctx, req, workflowConsistencyChecker)
|
||||
if errVerify == nil {
|
||||
return nil
|
||||
}
|
||||
switch errVerify.(type) {
|
||||
case *serviceerror.NotFound:
|
||||
case *serviceerror.WorkflowNotReady:
|
||||
if !firstWorkflowTaskMissing {
|
||||
return errVerify
|
||||
}
|
||||
default:
|
||||
return errVerify
|
||||
}
|
||||
if !req.GetResendChild() || !shardContext.GetConfig().EnableChildWorkflowResend() {
|
||||
return errVerify
|
||||
}
|
||||
|
||||
metricsHandler := shardContext.GetMetricsHandler()
|
||||
resend := func(ctx context.Context) error {
|
||||
metrics.ChildWorkflowResendAttempts.With(metricsHandler).Record(1)
|
||||
startTime := time.Now().UTC()
|
||||
err := resendChildAndVerify(
|
||||
ctx,
|
||||
req,
|
||||
workflowConsistencyChecker,
|
||||
shardContext,
|
||||
namespaceID,
|
||||
versionedTransition,
|
||||
versionHistories,
|
||||
errVerify,
|
||||
)
|
||||
metrics.ChildWorkflowResendLatency.With(metricsHandler).Record(time.Since(startTime))
|
||||
if err != nil && !isExpectedResendError(err) {
|
||||
metrics.ChildWorkflowResendFailures.With(metricsHandler).Record(1)
|
||||
}
|
||||
logResendFailure(shardContext, req, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if resendScheduler == nil {
|
||||
return resend(ctx)
|
||||
}
|
||||
|
||||
childKey := definition.NewWorkflowKey(req.NamespaceId, req.WorkflowExecution.WorkflowId, req.WorkflowExecution.RunId)
|
||||
resendCtx := rpc.CopyContextValues(shardContext.GetLifecycleContext(), ctx)
|
||||
submitResult := resendScheduler.TrySubmit(
|
||||
resendCtx,
|
||||
childKey,
|
||||
shardContext.GetConfig().ReplicationTaskApplyTimeout(),
|
||||
func(ctx context.Context) {
|
||||
_ = resend(ctx)
|
||||
},
|
||||
)
|
||||
switch submitResult {
|
||||
case workflowresend.SubmitResultAccepted:
|
||||
// Accepted work records its attempt when execution starts.
|
||||
case workflowresend.SubmitResultDuplicate:
|
||||
metrics.ChildWorkflowResendSkipped.With(metricsHandler).Record(1)
|
||||
case workflowresend.SubmitResultAtCapacity:
|
||||
metrics.ChildWorkflowResendLimited.With(metricsHandler).Record(1)
|
||||
default:
|
||||
// SubmitResultFailed and unknown values are admission failures.
|
||||
metrics.ChildWorkflowResendFailures.With(metricsHandler).Record(1)
|
||||
}
|
||||
// The submission result is intentionally used only for metrics. Preserve the verification error
|
||||
// so the durable standby task retries regardless of the admission outcome.
|
||||
return errVerify
|
||||
}
|
||||
|
||||
func verifyFirstWorkflowTaskScheduled(
|
||||
ctx context.Context,
|
||||
req *historyservice.VerifyFirstWorkflowTaskScheduledRequest,
|
||||
workflowConsistencyChecker api.WorkflowConsistencyChecker,
|
||||
) (
|
||||
versionedTransition *persistencespb.VersionedTransition,
|
||||
versionHistories *historyspb.VersionHistories,
|
||||
firstWorkflowTaskMissing bool,
|
||||
retError error,
|
||||
) {
|
||||
workflowLease, err := workflowConsistencyChecker.GetWorkflowLease(
|
||||
ctx,
|
||||
req.Clock,
|
||||
@@ -33,19 +128,88 @@ func Invoke(
|
||||
locks.PriorityLow,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, nil, false, err
|
||||
}
|
||||
defer func() { workflowLease.GetReleaseFn()(retError) }()
|
||||
|
||||
mutableState := workflowLease.GetMutableState()
|
||||
if !mutableState.IsWorkflowExecutionRunning() &&
|
||||
mutableState.GetExecutionState().State != enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE {
|
||||
return nil
|
||||
return nil, nil, false, nil
|
||||
}
|
||||
|
||||
if !mutableState.HadOrHasWorkflowTask() {
|
||||
return consts.ErrWorkflowNotReady
|
||||
executionInfo := mutableState.GetExecutionInfo()
|
||||
versionedTransition = transitionhistory.CopyVersionedTransition(
|
||||
transitionhistory.LastVersionedTransition(executionInfo.TransitionHistory),
|
||||
)
|
||||
versionHistories = versionhistory.CopyVersionHistories(executionInfo.VersionHistories)
|
||||
return versionedTransition, versionHistories, true, consts.ErrWorkflowNotReady
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil, nil, false, nil
|
||||
}
|
||||
|
||||
func logResendFailure(
|
||||
shardContext historyi.ShardContext,
|
||||
req *historyservice.VerifyFirstWorkflowTaskScheduledRequest,
|
||||
err error,
|
||||
) {
|
||||
if isExpectedResendError(err) {
|
||||
return
|
||||
}
|
||||
shardContext.GetThrottledLogger().Error(
|
||||
"Failed to resend child workflow for first workflow task verification",
|
||||
tag.WorkflowNamespaceID(req.GetNamespaceId()),
|
||||
tag.WorkflowID(req.WorkflowExecution.GetWorkflowId()),
|
||||
tag.WorkflowRunID(req.WorkflowExecution.GetRunId()),
|
||||
tag.Error(err),
|
||||
)
|
||||
}
|
||||
|
||||
func isExpectedResendError(err error) bool {
|
||||
if err == nil || common.IsContextCanceledErr(err) {
|
||||
return true
|
||||
}
|
||||
var notFoundErr *serviceerror.NotFound
|
||||
var workflowNotReadyErr *serviceerror.WorkflowNotReady
|
||||
var namespaceNotFoundErr *serviceerror.NamespaceNotFound
|
||||
return errors.As(err, ¬FoundErr) ||
|
||||
errors.As(err, &workflowNotReadyErr) ||
|
||||
errors.As(err, &namespaceNotFoundErr)
|
||||
}
|
||||
|
||||
func resendChildAndVerify(
|
||||
ctx context.Context,
|
||||
req *historyservice.VerifyFirstWorkflowTaskScheduledRequest,
|
||||
workflowConsistencyChecker api.WorkflowConsistencyChecker,
|
||||
shardContext historyi.ShardContext,
|
||||
namespaceID namespace.ID,
|
||||
versionedTransition *persistencespb.VersionedTransition,
|
||||
versionHistories *historyspb.VersionHistories,
|
||||
errVerify error,
|
||||
) error {
|
||||
result, err := workflowresend.SyncWorkflowStateFromSource(
|
||||
ctx,
|
||||
shardContext,
|
||||
namespaceID,
|
||||
req.WorkflowExecution,
|
||||
versionedTransition,
|
||||
versionHistories,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch result {
|
||||
case workflowresend.SyncWorkflowStateResultSourceNotFound:
|
||||
return nil
|
||||
case workflowresend.SyncWorkflowStateResultSkipped:
|
||||
return errVerify
|
||||
case workflowresend.SyncWorkflowStateResultApplied:
|
||||
_, _, _, err = verifyFirstWorkflowTaskScheduled(ctx, req, workflowConsistencyChecker)
|
||||
return err
|
||||
default:
|
||||
return fmt.Errorf("unknown workflow state sync result: %d", result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,12 +20,15 @@ import (
|
||||
workflowspb "go.temporal.io/server/api/workflow/v1"
|
||||
"go.temporal.io/server/common"
|
||||
"go.temporal.io/server/common/cluster"
|
||||
"go.temporal.io/server/common/definition"
|
||||
"go.temporal.io/server/common/log"
|
||||
"go.temporal.io/server/common/metrics"
|
||||
"go.temporal.io/server/common/metrics/metricstest"
|
||||
"go.temporal.io/server/common/payloads"
|
||||
"go.temporal.io/server/common/persistence"
|
||||
"go.temporal.io/server/common/primitives"
|
||||
"go.temporal.io/server/service/history/api"
|
||||
"go.temporal.io/server/service/history/api/workflowresend"
|
||||
"go.temporal.io/server/service/history/events"
|
||||
"go.temporal.io/server/service/history/hsm"
|
||||
historyi "go.temporal.io/server/service/history/interfaces"
|
||||
@@ -49,6 +52,7 @@ type (
|
||||
mockExecutionMgr *persistence.MockExecutionManager
|
||||
shardContext *shard.ContextTest
|
||||
workflowConsistencyChecker api.WorkflowConsistencyChecker
|
||||
resendScheduler *workflowresend.BoundedWorkflowScheduler
|
||||
|
||||
logger log.Logger
|
||||
}
|
||||
@@ -63,6 +67,12 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) SetupTest() {
|
||||
s.controller = gomock.NewController(s.T())
|
||||
|
||||
config := tests.NewDynamicConfig()
|
||||
config.WorkflowResendHostMaxInFlight = func() int { return 1 }
|
||||
s.resendScheduler = workflowresend.NewBoundedWorkflowScheduler(
|
||||
config.WorkflowResendHostMaxInFlight,
|
||||
log.NewNoopLogger(),
|
||||
metrics.NoopMetricsHandler,
|
||||
)
|
||||
s.shardContext = shard.NewTestContext(
|
||||
s.controller,
|
||||
&persistencespb.ShardInfo{
|
||||
@@ -95,24 +105,95 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) SetupTest() {
|
||||
}
|
||||
|
||||
func (s *VerifyFirstWorkflowTaskScheduledSuite) TearDownTest() {
|
||||
s.resendScheduler.InitiateShutdown()
|
||||
s.resendScheduler.WaitShutdown()
|
||||
s.controller.Finish()
|
||||
}
|
||||
|
||||
func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_WorkflowNotFound() {
|
||||
func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_WorkflowNotFound_ResendDisabled() {
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.NamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
|
||||
err := Invoke(context.Background(), request, s.workflowConsistencyChecker)
|
||||
err := Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler)
|
||||
s.IsType(&serviceerror.NotFound{}, err)
|
||||
}
|
||||
|
||||
func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_DoesNotResendUnrelatedWorkflowNotReady() {
|
||||
s.shardContext.GetConfig().EnableChildWorkflowResend = func() bool { return true }
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.NamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
workflowNotReadyErr := serviceerror.NewWorkflowNotReady("unrelated workflow state")
|
||||
workflowConsistencyChecker := api.NewMockWorkflowConsistencyChecker(s.controller)
|
||||
workflowConsistencyChecker.EXPECT().GetWorkflowLease(
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
definition.NewWorkflowKey(tests.NamespaceID.String(), tests.WorkflowID, tests.RunID),
|
||||
gomock.Any(),
|
||||
).Return(nil, workflowNotReadyErr)
|
||||
|
||||
err := Invoke(s.T().Context(), request, workflowConsistencyChecker, s.shardContext, s.resendScheduler)
|
||||
s.Same(workflowNotReadyErr, err)
|
||||
}
|
||||
|
||||
func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_HostAtCapacity() {
|
||||
s.shardContext.GetConfig().EnableChildWorkflowResend = func() bool { return true }
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
capture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(capture)
|
||||
s.shardContext.SetMetricsHandler(metricsHandler)
|
||||
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
defer close(release)
|
||||
s.Require().Equal(workflowresend.SubmitResultAccepted, s.resendScheduler.TrySubmit(
|
||||
s.T().Context(),
|
||||
definition.NewWorkflowKey("blocker namespace", "blocker workflow", "blocker run"),
|
||||
time.Minute,
|
||||
func(ctx context.Context) {
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
},
|
||||
))
|
||||
waitCtx, cancel := context.WithTimeout(s.T().Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
select {
|
||||
case <-started:
|
||||
case <-waitCtx.Done():
|
||||
s.T().Fatal("timed out waiting for host scheduler worker")
|
||||
}
|
||||
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.NamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
|
||||
err := Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler)
|
||||
s.Require().ErrorAs(err, new(*serviceerror.NotFound))
|
||||
s.Require().Len(capture.Snapshot()[metrics.ChildWorkflowResendLimited.Name()], 1)
|
||||
}
|
||||
|
||||
func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_WorkflowCompleted() {
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.NamespaceID.String(),
|
||||
@@ -137,11 +218,11 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched
|
||||
)
|
||||
s.NoError(err)
|
||||
|
||||
wfMs := workflow.TestCloneToProto(context.Background(), ms)
|
||||
wfMs := workflow.TestCloneToProto(s.T().Context(), ms)
|
||||
gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs}
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil)
|
||||
|
||||
err = Invoke(context.Background(), request, s.workflowConsistencyChecker)
|
||||
err = Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
@@ -170,11 +251,11 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched
|
||||
)
|
||||
s.NoError(err)
|
||||
|
||||
wfMs := workflow.TestCloneToProto(context.Background(), ms)
|
||||
wfMs := workflow.TestCloneToProto(s.T().Context(), ms)
|
||||
gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs}
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil)
|
||||
|
||||
err = Invoke(context.Background(), request, s.workflowConsistencyChecker)
|
||||
err = Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler)
|
||||
s.IsType(&serviceerror.WorkflowNotReady{}, err)
|
||||
}
|
||||
|
||||
@@ -197,11 +278,11 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched
|
||||
25*time.Second, 20*time.Second, 200*time.Second, nil, "identity")
|
||||
_, _ = ms.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL)
|
||||
|
||||
wfMs := workflow.TestCloneToProto(context.Background(), ms)
|
||||
wfMs := workflow.TestCloneToProto(s.T().Context(), ms)
|
||||
gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs}
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil)
|
||||
|
||||
err := Invoke(context.Background(), request, s.workflowConsistencyChecker)
|
||||
err := Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
@@ -249,11 +330,11 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched
|
||||
&workflowservice.RespondWorkflowTaskCompletedRequest{Identity: "some random identity"}, defaultWorkflowTaskCompletionLimits)
|
||||
ms.FlushBufferedEvents()
|
||||
|
||||
wfMs := workflow.TestCloneToProto(context.Background(), ms)
|
||||
wfMs := workflow.TestCloneToProto(s.T().Context(), ms)
|
||||
gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs}
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil)
|
||||
|
||||
err := Invoke(context.Background(), request, s.workflowConsistencyChecker)
|
||||
err := Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler)
|
||||
s.NoError(err)
|
||||
}
|
||||
|
||||
|
||||
214
service/history/api/workflowresend/scheduler.go
Normal file
214
service/history/api/workflowresend/scheduler.go
Normal file
@@ -0,0 +1,214 @@
|
||||
package workflowresend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.temporal.io/server/common/definition"
|
||||
"go.temporal.io/server/common/dynamicconfig"
|
||||
"go.temporal.io/server/common/log"
|
||||
"go.temporal.io/server/common/log/tag"
|
||||
"go.temporal.io/server/common/metrics"
|
||||
ctasks "go.temporal.io/server/common/tasks"
|
||||
)
|
||||
|
||||
const OperationName = "WorkflowResend"
|
||||
|
||||
// SubmitResult describes the outcome of scheduler admission.
|
||||
type SubmitResult int
|
||||
|
||||
const (
|
||||
SubmitResultFailed SubmitResult = 0
|
||||
SubmitResultAccepted SubmitResult = 1
|
||||
SubmitResultDuplicate SubmitResult = 2
|
||||
SubmitResultAtCapacity SubmitResult = 3
|
||||
)
|
||||
|
||||
// Scheduler runs workflow resend jobs asynchronously.
|
||||
type Scheduler interface {
|
||||
// TrySubmit reports whether the job was accepted, deduplicated, rejected at capacity, or failed
|
||||
// during submission.
|
||||
TrySubmit(
|
||||
ctx context.Context,
|
||||
key definition.WorkflowKey,
|
||||
timeout time.Duration,
|
||||
run func(context.Context),
|
||||
) SubmitResult
|
||||
}
|
||||
|
||||
// BoundedWorkflowScheduler deduplicates workflow resends and bounds their concurrency.
|
||||
type BoundedWorkflowScheduler struct {
|
||||
pool *ctasks.DynamicWorkerPoolScheduler
|
||||
|
||||
logger log.Logger
|
||||
metricsHandler metrics.Handler
|
||||
|
||||
mu sync.Mutex
|
||||
inFlight map[definition.WorkflowKey]struct{}
|
||||
}
|
||||
|
||||
var _ Scheduler = (*BoundedWorkflowScheduler)(nil)
|
||||
|
||||
// NewBoundedWorkflowScheduler creates a bounded workflow scheduler with no task buffer.
|
||||
func NewBoundedWorkflowScheduler(
|
||||
maxConcurrency dynamicconfig.IntPropertyFn,
|
||||
logger log.Logger,
|
||||
metricsHandler metrics.Handler,
|
||||
) *BoundedWorkflowScheduler {
|
||||
limiter := boundedWorkflowSchedulerLimiter{
|
||||
maxConcurrency: maxConcurrency,
|
||||
logger: logger,
|
||||
}
|
||||
return &BoundedWorkflowScheduler{
|
||||
pool: ctasks.NewDynamicWorkerPoolScheduler(
|
||||
limiter,
|
||||
metrics.NoopMetricsHandler,
|
||||
),
|
||||
logger: logger,
|
||||
metricsHandler: metricsHandler,
|
||||
inFlight: make(map[definition.WorkflowKey]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BoundedWorkflowScheduler) TrySubmit(
|
||||
ctx context.Context,
|
||||
key definition.WorkflowKey,
|
||||
timeout time.Duration,
|
||||
run func(context.Context),
|
||||
) (result SubmitResult) {
|
||||
var panicErr error
|
||||
defer log.CapturePanic(log.With(s.logger, workflowKeyTags(key)...), &panicErr)
|
||||
return s.trySubmit(ctx, key, timeout, run)
|
||||
}
|
||||
|
||||
func (s *BoundedWorkflowScheduler) trySubmit(
|
||||
ctx context.Context,
|
||||
key definition.WorkflowKey,
|
||||
timeout time.Duration,
|
||||
run func(context.Context),
|
||||
) SubmitResult {
|
||||
if !s.tryClaim(key) {
|
||||
return SubmitResultDuplicate
|
||||
}
|
||||
claimed := true
|
||||
var runnable *resendRunnable
|
||||
defer func() {
|
||||
if runnable != nil {
|
||||
runnable.Abort()
|
||||
} else if claimed {
|
||||
s.release(key)
|
||||
}
|
||||
}()
|
||||
|
||||
// The timeout starts at admission; this scheduler intentionally has no task buffer.
|
||||
jobCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
runnable = &resendRunnable{
|
||||
ctx: jobCtx,
|
||||
run: run,
|
||||
logger: log.With(s.logger, workflowKeyTags(key)...),
|
||||
cleanup: func() {
|
||||
cancel()
|
||||
s.release(key)
|
||||
},
|
||||
}
|
||||
if !s.pool.TrySubmit(runnable) {
|
||||
metrics.WorkflowResendSchedulerAtCapacity.With(s.metricsHandler).Record(1)
|
||||
return SubmitResultAtCapacity
|
||||
}
|
||||
runnable = nil
|
||||
claimed = false
|
||||
return SubmitResultAccepted
|
||||
}
|
||||
|
||||
func (s *BoundedWorkflowScheduler) tryClaim(key definition.WorkflowKey) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, ok := s.inFlight[key]; ok {
|
||||
return false
|
||||
}
|
||||
s.inFlight[key] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *BoundedWorkflowScheduler) release(key definition.WorkflowKey) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.inFlight, key)
|
||||
}
|
||||
|
||||
// InitiateShutdown cancels running jobs and aborts jobs waiting to run.
|
||||
func (s *BoundedWorkflowScheduler) InitiateShutdown() {
|
||||
s.pool.InitiateShutdown()
|
||||
}
|
||||
|
||||
// WaitShutdown waits for all scheduler goroutines to exit.
|
||||
func (s *BoundedWorkflowScheduler) WaitShutdown() {
|
||||
s.pool.WaitShutdown()
|
||||
}
|
||||
|
||||
type boundedWorkflowSchedulerLimiter struct {
|
||||
maxConcurrency dynamicconfig.IntPropertyFn
|
||||
logger log.Logger
|
||||
}
|
||||
|
||||
func (l boundedWorkflowSchedulerLimiter) Concurrency() (concurrency int) {
|
||||
var panicErr error
|
||||
defer func() {
|
||||
if panicErr != nil {
|
||||
concurrency = 0
|
||||
}
|
||||
}()
|
||||
defer log.CapturePanic(l.logger, &panicErr)
|
||||
return l.maxConcurrency()
|
||||
}
|
||||
|
||||
func (boundedWorkflowSchedulerLimiter) BufferSize() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
type resendRunnable struct {
|
||||
ctx context.Context
|
||||
run func(context.Context)
|
||||
logger log.Logger
|
||||
cleanup func()
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (r *resendRunnable) Run(shutdownCtx context.Context) {
|
||||
var panicErr error
|
||||
defer log.CapturePanic(r.logger, &panicErr)
|
||||
defer r.clean()
|
||||
|
||||
runCtx, cancel := context.WithCancel(r.ctx)
|
||||
stopShutdownCancellation := context.AfterFunc(shutdownCtx, cancel)
|
||||
defer func() {
|
||||
stopShutdownCancellation()
|
||||
cancel()
|
||||
}()
|
||||
if shutdownCtx.Err() != nil || runCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
r.run(runCtx)
|
||||
}
|
||||
|
||||
func (r *resendRunnable) Abort() {
|
||||
r.clean()
|
||||
}
|
||||
|
||||
func (r *resendRunnable) clean() {
|
||||
r.once.Do(func() {
|
||||
if r.cleanup != nil {
|
||||
r.cleanup()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func workflowKeyTags(key definition.WorkflowKey) []tag.Tag {
|
||||
return []tag.Tag{
|
||||
tag.WorkflowNamespaceID(key.NamespaceID),
|
||||
tag.WorkflowID(key.WorkflowID),
|
||||
tag.WorkflowRunID(key.RunID),
|
||||
}
|
||||
}
|
||||
365
service/history/api/workflowresend/scheduler_test.go
Normal file
365
service/history/api/workflowresend/scheduler_test.go
Normal file
@@ -0,0 +1,365 @@
|
||||
package workflowresend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.temporal.io/server/common/definition"
|
||||
"go.temporal.io/server/common/log"
|
||||
"go.temporal.io/server/common/metrics"
|
||||
"go.temporal.io/server/common/metrics/metricstest"
|
||||
"go.temporal.io/server/common/testing/await"
|
||||
)
|
||||
|
||||
const testJobTimeout = time.Minute
|
||||
|
||||
type panicDeadlineContext struct {
|
||||
context.Context
|
||||
}
|
||||
|
||||
func (panicDeadlineContext) Deadline() (time.Time, bool) {
|
||||
panic("test panic")
|
||||
}
|
||||
|
||||
func testWorkflowKey(workflowID string) definition.WorkflowKey {
|
||||
return definition.NewWorkflowKey("namespace", workflowID, "run")
|
||||
}
|
||||
|
||||
func TestSubmitResultZeroValueFailsSafe(t *testing.T) {
|
||||
var result SubmitResult
|
||||
require.Equal(t, SubmitResultFailed, result)
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerDeduplicatesAndReleasesWorkflow(t *testing.T) {
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
capture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(capture)
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int { return 2 }, log.NewNoopLogger(), metricsHandler)
|
||||
t.Cleanup(func() {
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
})
|
||||
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(ctx context.Context) {
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
close(finished)
|
||||
},
|
||||
))
|
||||
requireSignal(t, started)
|
||||
|
||||
var duplicateRuns atomic.Int32
|
||||
require.Equal(t, SubmitResultDuplicate, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(context.Context) { duplicateRuns.Add(1) },
|
||||
))
|
||||
require.Equal(t, SubmitResultDuplicate, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(context.Context) { duplicateRuns.Add(1) },
|
||||
))
|
||||
require.Zero(t, duplicateRuns.Load())
|
||||
require.Empty(t, capture.Snapshot()[metrics.WorkflowResendSchedulerAtCapacity.Name()])
|
||||
|
||||
differentWorkflowRan := make(chan struct{})
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("b"),
|
||||
testJobTimeout,
|
||||
func(context.Context) { close(differentWorkflowRan) },
|
||||
))
|
||||
requireSignal(t, differentWorkflowRan)
|
||||
|
||||
close(release)
|
||||
requireSignal(t, finished)
|
||||
resubmittedWorkflowRan := make(chan struct{})
|
||||
await.RequireTrue(t, func() bool {
|
||||
return scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(context.Context) { close(resubmittedWorkflowRan) },
|
||||
) == SubmitResultAccepted
|
||||
}, 5*time.Second, 10*time.Millisecond)
|
||||
requireSignal(t, resubmittedWorkflowRan)
|
||||
require.Zero(t, duplicateRuns.Load())
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerSharedConcurrencyRejectsAndReleasesWorkflow(t *testing.T) {
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
capture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(capture)
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metricsHandler)
|
||||
t.Cleanup(func() {
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
})
|
||||
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(ctx context.Context) {
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
close(finished)
|
||||
},
|
||||
))
|
||||
requireSignal(t, started)
|
||||
|
||||
var rejectedRuns atomic.Int32
|
||||
require.Equal(t, SubmitResultAtCapacity, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("b"),
|
||||
testJobTimeout,
|
||||
func(context.Context) { rejectedRuns.Add(1) },
|
||||
))
|
||||
require.Zero(t, rejectedRuns.Load())
|
||||
require.Len(t, capture.Snapshot()[metrics.WorkflowResendSchedulerAtCapacity.Name()], 1)
|
||||
require.Empty(t, capture.Snapshot()[metrics.DynamicWorkerPoolSchedulerRejectedTasks.Name()])
|
||||
|
||||
close(release)
|
||||
requireSignal(t, finished)
|
||||
resubmittedWorkflowRan := make(chan struct{})
|
||||
await.RequireTrue(t, func() bool {
|
||||
return scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("b"),
|
||||
testJobTimeout,
|
||||
func(context.Context) { close(resubmittedWorkflowRan) },
|
||||
) == SubmitResultAccepted
|
||||
}, 5*time.Second, 10*time.Millisecond)
|
||||
requireSignal(t, resubmittedWorkflowRan)
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerShutdownCancelsAndCleansUp(t *testing.T) {
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler)
|
||||
|
||||
started := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(ctx context.Context) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
close(finished)
|
||||
},
|
||||
))
|
||||
requireSignal(t, started)
|
||||
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
requireSignal(t, finished)
|
||||
require.Empty(t, scheduler.inFlight)
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerCallerCancellationReachesRunningJob(t *testing.T) {
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler)
|
||||
t.Cleanup(func() {
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
})
|
||||
|
||||
jobCtx, cancelJob := context.WithCancel(t.Context())
|
||||
started := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
jobCtx,
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(ctx context.Context) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
close(finished)
|
||||
},
|
||||
))
|
||||
requireSignal(t, started)
|
||||
|
||||
cancelJob()
|
||||
requireSignal(t, finished)
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerAbortCleansUp(t *testing.T) {
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler)
|
||||
scheduler.InitiateShutdown()
|
||||
|
||||
var runs atomic.Int32
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(context.Context) { runs.Add(1) },
|
||||
))
|
||||
scheduler.WaitShutdown()
|
||||
|
||||
require.Zero(t, runs.Load())
|
||||
require.Empty(t, scheduler.inFlight)
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerTimeoutReleasesWorkflow(t *testing.T) {
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler)
|
||||
t.Cleanup(func() {
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
})
|
||||
|
||||
finished := make(chan struct{})
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
10*time.Millisecond,
|
||||
func(ctx context.Context) {
|
||||
<-ctx.Done()
|
||||
close(finished)
|
||||
},
|
||||
))
|
||||
requireSignal(t, finished)
|
||||
|
||||
resubmittedWorkflowRan := make(chan struct{})
|
||||
await.RequireTrue(t, func() bool {
|
||||
return scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
testWorkflowKey("a"),
|
||||
testJobTimeout,
|
||||
func(context.Context) { close(resubmittedWorkflowRan) },
|
||||
) == SubmitResultAccepted
|
||||
}, 5*time.Second, 10*time.Millisecond)
|
||||
requireSignal(t, resubmittedWorkflowRan)
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerTaskPanicFailsOpen(t *testing.T) {
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler)
|
||||
t.Cleanup(func() {
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
})
|
||||
|
||||
key := testWorkflowKey("a")
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
key,
|
||||
testJobTimeout,
|
||||
func(context.Context) { panic("test panic") },
|
||||
))
|
||||
|
||||
resubmittedWorkflowRan := make(chan struct{})
|
||||
await.RequireTrue(t, func() bool {
|
||||
return scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
key,
|
||||
testJobTimeout,
|
||||
func(context.Context) { close(resubmittedWorkflowRan) },
|
||||
) == SubmitResultAccepted
|
||||
}, 5*time.Second, 10*time.Millisecond)
|
||||
requireSignal(t, resubmittedWorkflowRan)
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerLimitProviderPanicFailsOpen(t *testing.T) {
|
||||
var panicLimit atomic.Bool
|
||||
panicLimit.Store(true)
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int {
|
||||
if panicLimit.Swap(false) {
|
||||
panic("test panic")
|
||||
}
|
||||
return 1
|
||||
}, log.NewNoopLogger(), metrics.NoopMetricsHandler)
|
||||
t.Cleanup(func() {
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
})
|
||||
|
||||
key := testWorkflowKey("a")
|
||||
require.Equal(t, SubmitResultAtCapacity, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
key,
|
||||
testJobTimeout,
|
||||
func(context.Context) { require.FailNow(t, "rejected job ran") },
|
||||
))
|
||||
|
||||
resubmittedWorkflowRan := make(chan struct{})
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
key,
|
||||
testJobTimeout,
|
||||
func(context.Context) { close(resubmittedWorkflowRan) },
|
||||
))
|
||||
requireSignal(t, resubmittedWorkflowRan)
|
||||
}
|
||||
|
||||
func TestBoundedWorkflowSchedulerSubmitPanicFailsOpen(t *testing.T) {
|
||||
scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler)
|
||||
t.Cleanup(func() {
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
})
|
||||
|
||||
key := testWorkflowKey("a")
|
||||
require.Equal(t, SubmitResultFailed, scheduler.TrySubmit(
|
||||
panicDeadlineContext{Context: t.Context()},
|
||||
key,
|
||||
testJobTimeout,
|
||||
func(context.Context) { require.FailNow(t, "failed job ran") },
|
||||
))
|
||||
|
||||
resubmittedWorkflowRan := make(chan struct{})
|
||||
require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit(
|
||||
t.Context(),
|
||||
key,
|
||||
testJobTimeout,
|
||||
func(context.Context) { close(resubmittedWorkflowRan) },
|
||||
))
|
||||
requireSignal(t, resubmittedWorkflowRan)
|
||||
}
|
||||
|
||||
func TestResendRunnableDoesNotRunAfterShutdown(t *testing.T) {
|
||||
shutdownCtx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
var runs atomic.Int32
|
||||
var cleanups atomic.Int32
|
||||
runnable := &resendRunnable{
|
||||
ctx: t.Context(),
|
||||
run: func(context.Context) { runs.Add(1) },
|
||||
logger: log.NewNoopLogger(),
|
||||
cleanup: func() { cleanups.Add(1) },
|
||||
}
|
||||
runnable.Run(shutdownCtx)
|
||||
|
||||
require.Zero(t, runs.Load())
|
||||
require.Equal(t, int32(1), cleanups.Load())
|
||||
}
|
||||
|
||||
func requireSignal(t *testing.T, signal <-chan struct{}) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
select {
|
||||
case <-signal:
|
||||
case <-ctx.Done():
|
||||
require.FailNow(t, "timed out waiting for signal")
|
||||
}
|
||||
}
|
||||
119
service/history/api/workflowresend/sync_workflow_state.go
Normal file
119
service/history/api/workflowresend/sync_workflow_state.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package workflowresend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
commonpb "go.temporal.io/api/common/v1"
|
||||
"go.temporal.io/api/serviceerror"
|
||||
"go.temporal.io/server/api/adminservice/v1"
|
||||
historyspb "go.temporal.io/server/api/history/v1"
|
||||
persistencespb "go.temporal.io/server/api/persistence/v1"
|
||||
"go.temporal.io/server/chasm"
|
||||
"go.temporal.io/server/common"
|
||||
"go.temporal.io/server/common/namespace"
|
||||
"go.temporal.io/server/service/history/consts"
|
||||
historyi "go.temporal.io/server/service/history/interfaces"
|
||||
)
|
||||
|
||||
// SyncWorkflowStateResult describes the outcome of pulling and applying workflow state.
|
||||
type SyncWorkflowStateResult int
|
||||
|
||||
const (
|
||||
// Keep the zero value fail-safe for callers that do not initialize a result.
|
||||
SyncWorkflowStateResultSkipped SyncWorkflowStateResult = iota
|
||||
SyncWorkflowStateResultApplied
|
||||
SyncWorkflowStateResultSourceNotFound
|
||||
)
|
||||
|
||||
// SyncWorkflowStateFromSource pulls workflow state from the namespace's active cluster and applies
|
||||
// it locally if the namespace routing state remains unchanged for the duration of the RPC.
|
||||
func SyncWorkflowStateFromSource(
|
||||
ctx context.Context,
|
||||
shardContext historyi.ShardContext,
|
||||
namespaceID namespace.ID,
|
||||
execution *commonpb.WorkflowExecution,
|
||||
versionedTransition *persistencespb.VersionedTransition,
|
||||
versionHistories *historyspb.VersionHistories,
|
||||
onSourceResolved func(string),
|
||||
) (SyncWorkflowStateResult, error) {
|
||||
clusterMetadata := shardContext.GetClusterMetadata()
|
||||
currentClusterName := clusterMetadata.GetCurrentClusterName()
|
||||
namespaceRegistry := shardContext.GetNamespaceRegistry()
|
||||
namespaceEntry, err := namespaceRegistry.GetNamespaceByID(namespaceID)
|
||||
if err != nil {
|
||||
return SyncWorkflowStateResultSkipped, err
|
||||
}
|
||||
if !namespaceEntry.IsOnCluster(currentClusterName) {
|
||||
return SyncWorkflowStateResultSkipped, nil
|
||||
}
|
||||
|
||||
routingKey := namespace.RoutingKey{ID: execution.GetWorkflowId()}
|
||||
activeClusterName := namespaceEntry.ActiveClusterName(routingKey)
|
||||
if activeClusterName == currentClusterName {
|
||||
return SyncWorkflowStateResultSkipped, nil
|
||||
}
|
||||
|
||||
targetClusterInfo, ok := clusterMetadata.GetAllClusterInfo()[currentClusterName]
|
||||
if !ok {
|
||||
return SyncWorkflowStateResultSkipped, fmt.Errorf("current cluster %q is missing from cluster metadata", currentClusterName)
|
||||
}
|
||||
remoteAdminClient, err := shardContext.GetRemoteAdminClient(activeClusterName)
|
||||
if err != nil {
|
||||
return SyncWorkflowStateResultSkipped, err
|
||||
}
|
||||
if onSourceResolved != nil {
|
||||
onSourceResolved(activeClusterName)
|
||||
}
|
||||
|
||||
resp, err := remoteAdminClient.SyncWorkflowState(ctx, &adminservice.SyncWorkflowStateRequest{
|
||||
NamespaceId: namespaceID.String(),
|
||||
Execution: execution,
|
||||
ArchetypeId: chasm.WorkflowArchetypeID,
|
||||
VersionedTransition: versionedTransition,
|
||||
VersionHistories: versionHistories,
|
||||
TargetClusterId: int32(targetClusterInfo.InitialFailoverVersion),
|
||||
})
|
||||
if err != nil {
|
||||
if common.IsNotFoundError(err) {
|
||||
return SyncWorkflowStateResultSourceNotFound, nil
|
||||
}
|
||||
var failedPreconditionErr *serviceerror.FailedPrecondition
|
||||
if errors.As(err, &failedPreconditionErr) {
|
||||
return SyncWorkflowStateResultSkipped, nil
|
||||
}
|
||||
return SyncWorkflowStateResultSkipped, err
|
||||
}
|
||||
if resp == nil || resp.VersionedTransitionArtifact == nil {
|
||||
return SyncWorkflowStateResultSkipped, serviceerror.NewInternal("SyncWorkflowState returned an empty artifact")
|
||||
}
|
||||
|
||||
namespaceEntry, err = namespaceRegistry.GetNamespaceByID(namespaceID)
|
||||
if err != nil {
|
||||
var namespaceNotFoundErr *serviceerror.NamespaceNotFound
|
||||
if errors.As(err, &namespaceNotFoundErr) {
|
||||
return SyncWorkflowStateResultSkipped, nil
|
||||
}
|
||||
return SyncWorkflowStateResultSkipped, err
|
||||
}
|
||||
if !namespaceEntry.IsOnCluster(currentClusterName) ||
|
||||
namespaceEntry.ActiveClusterName(routingKey) != activeClusterName {
|
||||
return SyncWorkflowStateResultSkipped, nil
|
||||
}
|
||||
|
||||
engine, err := shardContext.GetEngine(ctx)
|
||||
if err != nil {
|
||||
return SyncWorkflowStateResultSkipped, err
|
||||
}
|
||||
if err := engine.ReplicateVersionedTransition(
|
||||
ctx,
|
||||
chasm.WorkflowArchetypeID,
|
||||
resp.VersionedTransitionArtifact,
|
||||
activeClusterName,
|
||||
); err != nil && !errors.Is(err, consts.ErrDuplicate) {
|
||||
return SyncWorkflowStateResultSkipped, err
|
||||
}
|
||||
|
||||
return SyncWorkflowStateResultApplied, nil
|
||||
}
|
||||
324
service/history/api/workflowresend/sync_workflow_state_test.go
Normal file
324
service/history/api/workflowresend/sync_workflow_state_test.go
Normal file
@@ -0,0 +1,324 @@
|
||||
package workflowresend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
commonpb "go.temporal.io/api/common/v1"
|
||||
"go.temporal.io/api/serviceerror"
|
||||
"go.temporal.io/server/api/adminservice/v1"
|
||||
"go.temporal.io/server/api/adminservicemock/v1"
|
||||
historyspb "go.temporal.io/server/api/history/v1"
|
||||
persistencespb "go.temporal.io/server/api/persistence/v1"
|
||||
replicationspb "go.temporal.io/server/api/replication/v1"
|
||||
"go.temporal.io/server/chasm"
|
||||
"go.temporal.io/server/common/cluster"
|
||||
"go.temporal.io/server/common/namespace"
|
||||
"go.temporal.io/server/common/testing/protomock"
|
||||
"go.temporal.io/server/service/history/consts"
|
||||
historyi "go.temporal.io/server/service/history/interfaces"
|
||||
"go.uber.org/mock/gomock"
|
||||
)
|
||||
|
||||
const (
|
||||
syncTestNamespaceID = namespace.ID("sync-namespace-id")
|
||||
syncTestWorkflowID = "sync-workflow-id"
|
||||
syncTestRunID = "sync-run-id"
|
||||
syncTestCurrentCluster = "sync-cluster-b"
|
||||
syncTestSourceCluster = "sync-cluster-a"
|
||||
syncTestAlternativeSourceCluster = "sync-cluster-c"
|
||||
syncTestCurrentFailoverVersion = int64(22)
|
||||
)
|
||||
|
||||
type syncWorkflowStateFixture struct {
|
||||
shard *historyi.MockShardContext
|
||||
registry *namespace.MockRegistry
|
||||
cluster *cluster.MockMetadata
|
||||
remoteClient *adminservicemock.MockAdminServiceClient
|
||||
engine *historyi.MockEngine
|
||||
execution *commonpb.WorkflowExecution
|
||||
transition *persistencespb.VersionedTransition
|
||||
versionHistory *historyspb.VersionHistories
|
||||
}
|
||||
|
||||
func TestSyncWorkflowStateResultZeroValueSkips(t *testing.T) {
|
||||
var result SyncWorkflowStateResult
|
||||
require.Equal(t, SyncWorkflowStateResultSkipped, result)
|
||||
}
|
||||
|
||||
func TestSyncWorkflowStateFromSource_AppliesArtifact(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
applyErr error
|
||||
}{
|
||||
{name: "applied"},
|
||||
{name: "duplicate is applied", applyErr: consts.ErrDuplicate},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
artifact := &replicationspb.VersionedTransitionArtifact{}
|
||||
f.expectEligibleNamespace()
|
||||
f.remoteClient.EXPECT().SyncWorkflowState(
|
||||
gomock.Any(),
|
||||
protomock.Eq(f.expectedRequest()),
|
||||
).Return(&adminservice.SyncWorkflowStateResponse{
|
||||
VersionedTransitionArtifact: artifact,
|
||||
}, nil)
|
||||
f.registry.EXPECT().GetNamespaceByID(syncTestNamespaceID).Return(
|
||||
syncTestNamespace(syncTestSourceCluster, syncTestSourceCluster, syncTestCurrentCluster),
|
||||
nil,
|
||||
)
|
||||
f.shard.EXPECT().GetEngine(gomock.Any()).Return(f.engine, nil)
|
||||
f.engine.EXPECT().ReplicateVersionedTransition(
|
||||
gomock.Any(),
|
||||
chasm.WorkflowArchetypeID,
|
||||
artifact,
|
||||
syncTestSourceCluster,
|
||||
).Return(test.applyErr)
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, SyncWorkflowStateResultApplied, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncWorkflowStateFromSource_SourceResponse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
rpcErr error
|
||||
result SyncWorkflowStateResult
|
||||
}{
|
||||
{
|
||||
name: "source workflow not found",
|
||||
rpcErr: serviceerror.NewNotFound("workflow not found"),
|
||||
result: SyncWorkflowStateResultSourceNotFound,
|
||||
},
|
||||
{
|
||||
name: "transition history unsupported",
|
||||
rpcErr: serviceerror.NewFailedPrecondition("transition history disabled"),
|
||||
result: SyncWorkflowStateResultSkipped,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
f.expectEligibleNamespace()
|
||||
f.remoteClient.EXPECT().SyncWorkflowState(
|
||||
gomock.Any(),
|
||||
protomock.Eq(f.expectedRequest()),
|
||||
).Return(nil, test.rpcErr)
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.result, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncWorkflowStateFromSource_SkipsWhenNamespaceBecomesIneligible(t *testing.T) {
|
||||
t.Run("removed from current cluster before request", func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
f.expectNamespaceLookup(
|
||||
syncTestNamespace(syncTestSourceCluster, syncTestSourceCluster),
|
||||
nil,
|
||||
)
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, SyncWorkflowStateResultSkipped, result)
|
||||
})
|
||||
|
||||
t.Run("active on current cluster before request", func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
f.expectNamespaceLookup(
|
||||
syncTestNamespace(syncTestCurrentCluster, syncTestSourceCluster, syncTestCurrentCluster),
|
||||
nil,
|
||||
)
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, SyncWorkflowStateResultSkipped, result)
|
||||
})
|
||||
|
||||
t.Run("namespace deleted before request", func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
f.expectNamespaceLookup(nil, serviceerror.NewNamespaceNotFound(syncTestNamespaceID.String()))
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.ErrorAs(t, err, new(*serviceerror.NamespaceNotFound))
|
||||
require.Equal(t, SyncWorkflowStateResultSkipped, result)
|
||||
})
|
||||
|
||||
postRequestStates := []struct {
|
||||
name string
|
||||
entry *namespace.Namespace
|
||||
lookupErr error
|
||||
}{
|
||||
{
|
||||
name: "namespace removed from current cluster during request",
|
||||
entry: syncTestNamespace(syncTestSourceCluster, syncTestSourceCluster),
|
||||
},
|
||||
{
|
||||
name: "namespace becomes active locally during request",
|
||||
entry: syncTestNamespace(syncTestCurrentCluster, syncTestSourceCluster, syncTestCurrentCluster),
|
||||
},
|
||||
{
|
||||
name: "source changes during request",
|
||||
entry: syncTestNamespace(syncTestAlternativeSourceCluster, syncTestAlternativeSourceCluster, syncTestCurrentCluster),
|
||||
},
|
||||
{
|
||||
name: "namespace deleted during request",
|
||||
lookupErr: serviceerror.NewNamespaceNotFound(syncTestNamespaceID.String()),
|
||||
},
|
||||
}
|
||||
for _, postRequestState := range postRequestStates {
|
||||
t.Run(postRequestState.name, func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
f.expectEligibleNamespace()
|
||||
f.remoteClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Return(
|
||||
&adminservice.SyncWorkflowStateResponse{
|
||||
VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{},
|
||||
},
|
||||
nil,
|
||||
)
|
||||
f.registry.EXPECT().GetNamespaceByID(syncTestNamespaceID).Return(
|
||||
postRequestState.entry,
|
||||
postRequestState.lookupErr,
|
||||
)
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, SyncWorkflowStateResultSkipped, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncWorkflowStateFromSource_PropagatesErrors(t *testing.T) {
|
||||
t.Run("initial namespace lookup", func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
testErr := errors.New("namespace lookup failed")
|
||||
f.expectNamespaceLookup(nil, testErr)
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.ErrorIs(t, err, testErr)
|
||||
require.Equal(t, SyncWorkflowStateResultSkipped, result)
|
||||
})
|
||||
|
||||
t.Run("sync request", func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
testErr := errors.New("sync request failed")
|
||||
f.expectEligibleNamespace()
|
||||
f.remoteClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Return(nil, testErr)
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.ErrorIs(t, err, testErr)
|
||||
require.Equal(t, SyncWorkflowStateResultSkipped, result)
|
||||
})
|
||||
|
||||
t.Run("refreshed namespace lookup", func(t *testing.T) {
|
||||
f := newSyncWorkflowStateFixture(t)
|
||||
testErr := errors.New("namespace refresh failed")
|
||||
f.expectEligibleNamespace()
|
||||
f.remoteClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Return(
|
||||
&adminservice.SyncWorkflowStateResponse{
|
||||
VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{},
|
||||
},
|
||||
nil,
|
||||
)
|
||||
f.registry.EXPECT().GetNamespaceByID(syncTestNamespaceID).Return(nil, testErr)
|
||||
|
||||
result, err := f.sync(t.Context())
|
||||
|
||||
require.ErrorIs(t, err, testErr)
|
||||
require.Equal(t, SyncWorkflowStateResultSkipped, result)
|
||||
})
|
||||
}
|
||||
|
||||
func newSyncWorkflowStateFixture(t *testing.T) *syncWorkflowStateFixture {
|
||||
controller := gomock.NewController(t)
|
||||
return &syncWorkflowStateFixture{
|
||||
shard: historyi.NewMockShardContext(controller),
|
||||
registry: namespace.NewMockRegistry(controller),
|
||||
cluster: cluster.NewMockMetadata(controller),
|
||||
remoteClient: adminservicemock.NewMockAdminServiceClient(controller),
|
||||
engine: historyi.NewMockEngine(controller),
|
||||
execution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: syncTestWorkflowID,
|
||||
RunId: syncTestRunID,
|
||||
},
|
||||
transition: &persistencespb.VersionedTransition{
|
||||
NamespaceFailoverVersion: 11,
|
||||
TransitionCount: 12,
|
||||
},
|
||||
versionHistory: &historyspb.VersionHistories{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *syncWorkflowStateFixture) expectNamespaceLookup(entry *namespace.Namespace, err error) {
|
||||
f.shard.EXPECT().GetClusterMetadata().Return(f.cluster)
|
||||
f.cluster.EXPECT().GetCurrentClusterName().Return(syncTestCurrentCluster)
|
||||
f.shard.EXPECT().GetNamespaceRegistry().Return(f.registry)
|
||||
f.registry.EXPECT().GetNamespaceByID(syncTestNamespaceID).Return(entry, err)
|
||||
}
|
||||
|
||||
func (f *syncWorkflowStateFixture) expectEligibleNamespace() {
|
||||
f.expectNamespaceLookup(
|
||||
syncTestNamespace(syncTestSourceCluster, syncTestSourceCluster, syncTestCurrentCluster),
|
||||
nil,
|
||||
)
|
||||
f.cluster.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{
|
||||
syncTestCurrentCluster: {
|
||||
InitialFailoverVersion: syncTestCurrentFailoverVersion,
|
||||
},
|
||||
})
|
||||
f.shard.EXPECT().GetRemoteAdminClient(syncTestSourceCluster).Return(f.remoteClient, nil)
|
||||
}
|
||||
|
||||
func (f *syncWorkflowStateFixture) expectedRequest() *adminservice.SyncWorkflowStateRequest {
|
||||
return &adminservice.SyncWorkflowStateRequest{
|
||||
NamespaceId: syncTestNamespaceID.String(),
|
||||
Execution: f.execution,
|
||||
ArchetypeId: chasm.WorkflowArchetypeID,
|
||||
VersionedTransition: f.transition,
|
||||
VersionHistories: f.versionHistory,
|
||||
TargetClusterId: int32(syncTestCurrentFailoverVersion),
|
||||
}
|
||||
}
|
||||
|
||||
func (f *syncWorkflowStateFixture) sync(ctx context.Context) (SyncWorkflowStateResult, error) {
|
||||
return SyncWorkflowStateFromSource(
|
||||
ctx,
|
||||
f.shard,
|
||||
syncTestNamespaceID,
|
||||
f.execution,
|
||||
f.transition,
|
||||
f.versionHistory,
|
||||
nil,
|
||||
)
|
||||
}
|
||||
|
||||
func syncTestNamespace(activeCluster string, clusters ...string) *namespace.Namespace {
|
||||
return namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: syncTestNamespaceID.String(), Name: "sync-test-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: activeCluster,
|
||||
Clusters: clusters,
|
||||
},
|
||||
1,
|
||||
)
|
||||
}
|
||||
@@ -288,7 +288,8 @@ type Config struct {
|
||||
// The following is used by the new RPC replication stack
|
||||
ReplicationTaskApplyTimeout dynamicconfig.DurationPropertyFn
|
||||
EnableAsyncParentWorkflowResend dynamicconfig.BoolPropertyFn
|
||||
ParentWorkflowResendMaxInFlight dynamicconfig.IntPropertyFn
|
||||
EnableChildWorkflowResend dynamicconfig.BoolPropertyFn
|
||||
WorkflowResendHostMaxInFlight dynamicconfig.IntPropertyFn
|
||||
ReplicationTaskFetcherParallelism dynamicconfig.IntPropertyFn
|
||||
ReplicationTaskFetcherAggregationInterval dynamicconfig.DurationPropertyFn
|
||||
ReplicationTaskFetcherTimerJitterCoefficient dynamicconfig.FloatPropertyFn
|
||||
@@ -731,7 +732,8 @@ func NewConfig(
|
||||
|
||||
ReplicationTaskApplyTimeout: dynamicconfig.ReplicationTaskApplyTimeout.Get(dc),
|
||||
EnableAsyncParentWorkflowResend: dynamicconfig.EnableAsyncParentWorkflowResend.Get(dc),
|
||||
ParentWorkflowResendMaxInFlight: dynamicconfig.ParentWorkflowResendMaxInFlight.Get(dc),
|
||||
EnableChildWorkflowResend: dynamicconfig.EnableChildWorkflowResend.Get(dc),
|
||||
WorkflowResendHostMaxInFlight: dynamicconfig.WorkflowResendHostMaxInFlight.Get(dc),
|
||||
ReplicationTaskFetcherParallelism: dynamicconfig.ReplicationTaskFetcherParallelism.Get(dc),
|
||||
ReplicationTaskFetcherAggregationInterval: dynamicconfig.ReplicationTaskFetcherAggregationInterval.Get(dc),
|
||||
ReplicationTaskFetcherTimerJitterCoefficient: dynamicconfig.ReplicationTaskFetcherTimerJitterCoefficient.Get(dc),
|
||||
|
||||
@@ -42,6 +42,7 @@ import (
|
||||
hsmnexusworkflow "go.temporal.io/server/components/nexusoperations/workflow"
|
||||
"go.temporal.io/server/service"
|
||||
"go.temporal.io/server/service/history/api"
|
||||
"go.temporal.io/server/service/history/api/workflowresend"
|
||||
"go.temporal.io/server/service/history/archival"
|
||||
"go.temporal.io/server/service/history/configs"
|
||||
"go.temporal.io/server/service/history/consts"
|
||||
@@ -92,6 +93,7 @@ var Module = fx.Options(
|
||||
service.PersistenceLazyLoadedServiceResolverModule,
|
||||
fx.Provide(ServiceResolverProvider),
|
||||
fx.Provide(EventNotifierProvider),
|
||||
fx.Provide(WorkflowResendSchedulerProvider),
|
||||
fx.Provide(HistoryEngineFactoryProvider),
|
||||
fx.Provide(HandlerProvider),
|
||||
fx.Provide(HistoryServiceServerProvider),
|
||||
@@ -488,6 +490,46 @@ func ServiceLifetimeHooks(lc fx.Lifecycle, svc *Service) {
|
||||
lc.Append(fx.StartStopHook(svc.Start, svc.Stop))
|
||||
}
|
||||
|
||||
func WorkflowResendSchedulerProvider(
|
||||
lc fx.Lifecycle,
|
||||
serviceConfig *configs.Config,
|
||||
metricsHandler metrics.Handler,
|
||||
logger log.ThrottledLogger,
|
||||
) workflowresend.Scheduler {
|
||||
schedulerLogger := log.With(
|
||||
logger,
|
||||
tag.ComponentTaskScheduler,
|
||||
tag.ScopeHost,
|
||||
tag.Operation(workflowresend.OperationName),
|
||||
)
|
||||
workflowResendScheduler := workflowresend.NewBoundedWorkflowScheduler(
|
||||
serviceConfig.WorkflowResendHostMaxInFlight,
|
||||
schedulerLogger,
|
||||
metricsHandler,
|
||||
)
|
||||
lc.Append(fx.Hook{
|
||||
OnStop: func(ctx context.Context) error {
|
||||
workflowResendScheduler.InitiateShutdown()
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||
defer cancel()
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
workflowResendScheduler.WaitShutdown()
|
||||
close(stopped)
|
||||
}()
|
||||
select {
|
||||
case <-stopped:
|
||||
return nil
|
||||
case <-shutdownCtx.Done():
|
||||
schedulerLogger.Warn("Workflow resend scheduler timed out during shutdown", tag.Error(shutdownCtx.Err()))
|
||||
return shutdownCtx.Err()
|
||||
}
|
||||
},
|
||||
})
|
||||
return workflowResendScheduler
|
||||
}
|
||||
|
||||
func ReplicationProgressCacheProvider(
|
||||
serviceConfig *configs.Config,
|
||||
logger log.Logger,
|
||||
|
||||
@@ -85,6 +85,7 @@ import (
|
||||
"go.temporal.io/server/service/history/api/updateworkflowoptions"
|
||||
"go.temporal.io/server/service/history/api/verifychildworkflowcompletionrecorded"
|
||||
"go.temporal.io/server/service/history/api/verifyfirstworkflowtaskscheduled"
|
||||
"go.temporal.io/server/service/history/api/workflowresend"
|
||||
"go.temporal.io/server/service/history/circuitbreakerpool"
|
||||
"go.temporal.io/server/service/history/configs"
|
||||
"go.temporal.io/server/service/history/consts"
|
||||
@@ -103,6 +104,13 @@ import (
|
||||
)
|
||||
|
||||
type (
|
||||
engineOptions struct {
|
||||
workflowResendScheduler workflowresend.Scheduler
|
||||
}
|
||||
|
||||
// EngineOption adds optional history engine dependencies without adding required parameters.
|
||||
EngineOption func(*engineOptions)
|
||||
|
||||
historyEngineImpl struct {
|
||||
status int32
|
||||
currentClusterName string
|
||||
@@ -137,7 +145,7 @@ type (
|
||||
workflowDeleteManager deletemanager.DeleteManager
|
||||
serializer serialization.Serializer
|
||||
workflowConsistencyChecker api.WorkflowConsistencyChecker
|
||||
parentResends verifychildworkflowcompletionrecorded.InFlightResends
|
||||
workflowResendScheduler workflowresend.Scheduler
|
||||
chasmEngine chasm.Engine
|
||||
versionChecker headers.VersionChecker
|
||||
versionCache worker_versioning.VersionMembershipAndReactivationStatusCache
|
||||
@@ -155,6 +163,21 @@ type (
|
||||
}
|
||||
)
|
||||
|
||||
// WithWorkflowResendScheduler configures the host-level workflow resend scheduler.
|
||||
func WithWorkflowResendScheduler(scheduler workflowresend.Scheduler) EngineOption {
|
||||
return func(options *engineOptions) {
|
||||
options.workflowResendScheduler = scheduler
|
||||
}
|
||||
}
|
||||
|
||||
func applyEngineOptions(options []EngineOption) engineOptions {
|
||||
var result engineOptions
|
||||
for _, option := range options {
|
||||
option(&result)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// NewEngineWithShardContext creates an instance of history engine
|
||||
func NewEngineWithShardContext(
|
||||
shard historyi.ShardContext,
|
||||
@@ -185,7 +208,9 @@ func NewEngineWithShardContext(
|
||||
persistenceRateLimiter quotas.RequestRateLimiter,
|
||||
testHooks testhooks.TestHooks,
|
||||
chasmEngine chasm.Engine,
|
||||
options ...EngineOption,
|
||||
) historyi.Engine {
|
||||
engineOptions := applyEngineOptions(options)
|
||||
currentClusterName := shard.GetClusterMetadata().GetCurrentClusterName()
|
||||
|
||||
logger := shard.GetLogger()
|
||||
@@ -220,6 +245,7 @@ func NewEngineWithShardContext(
|
||||
eventNotifier: eventNotifier,
|
||||
fastForwardNotifier: notification.NewTimeSkippingFastForwardNotifier(),
|
||||
config: config,
|
||||
workflowResendScheduler: engineOptions.workflowResendScheduler,
|
||||
sdkClientFactory: sdkClientFactory,
|
||||
matchingClient: matchingClient,
|
||||
rawMatchingClient: rawMatchingClient,
|
||||
@@ -560,7 +586,13 @@ func (e *historyEngineImpl) VerifyFirstWorkflowTaskScheduled(
|
||||
ctx context.Context,
|
||||
request *historyservice.VerifyFirstWorkflowTaskScheduledRequest,
|
||||
) (retError error) {
|
||||
return verifyfirstworkflowtaskscheduled.Invoke(ctx, request, e.workflowConsistencyChecker)
|
||||
return verifyfirstworkflowtaskscheduled.Invoke(
|
||||
ctx,
|
||||
request,
|
||||
e.workflowConsistencyChecker,
|
||||
e.shardContext,
|
||||
e.workflowResendScheduler,
|
||||
)
|
||||
}
|
||||
|
||||
// RecordWorkflowTaskStarted starts a workflow task
|
||||
@@ -743,7 +775,13 @@ func (e *historyEngineImpl) VerifyChildExecutionCompletionRecorded(
|
||||
ctx context.Context,
|
||||
req *historyservice.VerifyChildExecutionCompletionRecordedRequest,
|
||||
) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) {
|
||||
return verifychildworkflowcompletionrecorded.Invoke(ctx, req, e.workflowConsistencyChecker, e.shardContext, &e.parentResends)
|
||||
return verifychildworkflowcompletionrecorded.Invoke(
|
||||
ctx,
|
||||
req,
|
||||
e.workflowConsistencyChecker,
|
||||
e.shardContext,
|
||||
e.workflowResendScheduler,
|
||||
)
|
||||
}
|
||||
|
||||
func (e *historyEngineImpl) ReplicateEventsV2(
|
||||
|
||||
@@ -37,6 +37,7 @@ import (
|
||||
"go.temporal.io/server/common/log"
|
||||
"go.temporal.io/server/common/log/tag"
|
||||
"go.temporal.io/server/common/metrics"
|
||||
"go.temporal.io/server/common/metrics/metricstest"
|
||||
"go.temporal.io/server/common/namespace"
|
||||
"go.temporal.io/server/common/payloads"
|
||||
"go.temporal.io/server/common/persistence"
|
||||
@@ -49,11 +50,13 @@ import (
|
||||
serviceerrors "go.temporal.io/server/common/serviceerror"
|
||||
"go.temporal.io/server/common/tasktoken"
|
||||
"go.temporal.io/server/common/testing/await"
|
||||
"go.temporal.io/server/common/testing/protomock"
|
||||
"go.temporal.io/server/common/testing/protorequire"
|
||||
"go.temporal.io/server/common/testing/testvars"
|
||||
"go.temporal.io/server/common/util"
|
||||
"go.temporal.io/server/common/wideevents"
|
||||
"go.temporal.io/server/service/history/api"
|
||||
"go.temporal.io/server/service/history/api/workflowresend"
|
||||
"go.temporal.io/server/service/history/configs"
|
||||
"go.temporal.io/server/service/history/consts"
|
||||
"go.temporal.io/server/service/history/events"
|
||||
@@ -97,6 +100,7 @@ type (
|
||||
workflowCache wcache.Cache
|
||||
historyEngine *historyEngineImpl
|
||||
mockExecutionMgr *persistence.MockExecutionManager
|
||||
resendScheduler *workflowresend.BoundedWorkflowScheduler
|
||||
|
||||
config *configs.Config
|
||||
logger *log.MockLogger
|
||||
@@ -147,6 +151,16 @@ func (s *engine2Suite) SetupTest() {
|
||||
|
||||
s.config = tests.NewDynamicConfig()
|
||||
s.parentChildEventCapture = &parentChildEventCapture{}
|
||||
resendScheduler := workflowresend.NewBoundedWorkflowScheduler(
|
||||
func() int { return s.config.WorkflowResendHostMaxInFlight() },
|
||||
log.NewNoopLogger(),
|
||||
metrics.NoopMetricsHandler,
|
||||
)
|
||||
s.T().Cleanup(func() {
|
||||
resendScheduler.InitiateShutdown()
|
||||
resendScheduler.WaitShutdown()
|
||||
})
|
||||
s.resendScheduler = resendScheduler
|
||||
mockShard := shard.NewTestContext(
|
||||
s.controller,
|
||||
&persistencespb.ShardInfo{
|
||||
@@ -232,6 +246,7 @@ func (s *engine2Suite) SetupTest() {
|
||||
log.NewNoopLogger(),
|
||||
),
|
||||
workflowConsistencyChecker: api.NewWorkflowConsistencyChecker(mockShard, s.workflowCache),
|
||||
workflowResendScheduler: s.resendScheduler,
|
||||
persistenceVisibilityMgr: s.mockVisibilityManager,
|
||||
nDCWorkflowStateReplicator: s.mockWorkflowStateReplicator,
|
||||
workerDeploymentClient: noopWorkerDeploymentClient{},
|
||||
@@ -249,6 +264,8 @@ func (s *engine2Suite) SetupSubTest() {
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TearDownTest() {
|
||||
s.resendScheduler.InitiateShutdown()
|
||||
s.resendScheduler.WaitShutdown()
|
||||
s.controller.Finish()
|
||||
s.mockShard.StopForTest()
|
||||
}
|
||||
@@ -2665,6 +2682,388 @@ func (s *engine2Suite) TestRecordChildExecutionCompleted_MissingChildStartedEven
|
||||
}
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildAsync() {
|
||||
s.config.EnableChildWorkflowResend = func() bool { return true }
|
||||
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.NamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
|
||||
mockClusterMetadata := cluster.NewMockMetadata(s.controller)
|
||||
mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo)
|
||||
mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes()
|
||||
s.mockShard.SetClusterMetadata(mockClusterMetadata)
|
||||
|
||||
syncRequest := &adminservice.SyncWorkflowStateRequest{
|
||||
NamespaceId: request.NamespaceId,
|
||||
Execution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: request.WorkflowExecution.WorkflowId,
|
||||
RunId: request.WorkflowExecution.RunId,
|
||||
},
|
||||
TargetClusterId: int32(cluster.TestAlternativeClusterInitialFailoverVersion),
|
||||
ArchetypeId: chasm.WorkflowArchetypeID,
|
||||
}
|
||||
syncResponse := &adminservice.SyncWorkflowStateResponse{
|
||||
VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{
|
||||
StateAttributes: &replicationspb.VersionedTransitionArtifact_SyncWorkflowStateSnapshotAttributes{
|
||||
SyncWorkflowStateSnapshotAttributes: &replicationspb.SyncWorkflowStateSnapshotAttributes{
|
||||
State: &persistencespb.WorkflowMutableState{},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), syncRequest).Return(syncResponse, nil)
|
||||
s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition(
|
||||
gomock.Any(),
|
||||
chasm.WorkflowArchetypeID,
|
||||
syncResponse.VersionedTransitionArtifact,
|
||||
cluster.TestCurrentClusterName,
|
||||
).Return(nil)
|
||||
|
||||
ms := workflow.TestGlobalMutableState(
|
||||
s.historyEngine.shardContext,
|
||||
s.mockEventsCache,
|
||||
log.NewTestLogger(),
|
||||
tests.Version,
|
||||
tests.WorkflowID,
|
||||
tests.RunID,
|
||||
)
|
||||
addWorkflowExecutionStartedEvent(
|
||||
ms,
|
||||
request.WorkflowExecution,
|
||||
"wType",
|
||||
"testTaskQueue",
|
||||
payloads.EncodeString("input"),
|
||||
25*time.Second,
|
||||
20*time.Second,
|
||||
200*time.Second,
|
||||
"identity",
|
||||
)
|
||||
_, err := ms.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL)
|
||||
s.NoError(err)
|
||||
|
||||
resendVerified := make(chan struct{})
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn(
|
||||
func(context.Context, *persistence.GetWorkflowExecutionRequest) (*persistence.GetWorkflowExecutionResponse, error) {
|
||||
close(resendVerified)
|
||||
return &persistence.GetWorkflowExecutionResponse{State: workflow.TestCloneToProto(s.T().Context(), ms)}, nil
|
||||
},
|
||||
)
|
||||
|
||||
err = s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request)
|
||||
var notFound *serviceerror.NotFound
|
||||
s.ErrorAs(err, ¬Found)
|
||||
|
||||
select {
|
||||
case <-resendVerified:
|
||||
case <-time.After(10 * time.Second):
|
||||
s.Fail("background child resend was not re-verified")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_NilSchedulerResendsSynchronously() {
|
||||
s.config.EnableChildWorkflowResend = func() bool { return true }
|
||||
scheduler := s.historyEngine.workflowResendScheduler
|
||||
s.historyEngine.workflowResendScheduler = nil
|
||||
s.T().Cleanup(func() {
|
||||
s.historyEngine.workflowResendScheduler = scheduler
|
||||
})
|
||||
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.NamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
|
||||
mockClusterMetadata := cluster.NewMockMetadata(s.controller)
|
||||
mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo)
|
||||
mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes()
|
||||
s.mockShard.SetClusterMetadata(mockClusterMetadata)
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).
|
||||
Return(nil, serviceerror.NewUnavailable("source cluster unavailable"))
|
||||
|
||||
err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request)
|
||||
s.Require().ErrorAs(err, new(*serviceerror.Unavailable))
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendsChildWhenWorkflowNotReady() {
|
||||
s.config.EnableChildWorkflowResend = func() bool { return true }
|
||||
scheduler := s.historyEngine.workflowResendScheduler
|
||||
s.historyEngine.workflowResendScheduler = nil
|
||||
s.T().Cleanup(func() {
|
||||
s.historyEngine.workflowResendScheduler = scheduler
|
||||
})
|
||||
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.NamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
|
||||
ms := workflow.TestGlobalMutableState(
|
||||
s.historyEngine.shardContext,
|
||||
s.mockEventsCache,
|
||||
log.NewTestLogger(),
|
||||
tests.Version,
|
||||
tests.WorkflowID,
|
||||
tests.RunID,
|
||||
)
|
||||
addWorkflowExecutionStartedEvent(
|
||||
ms,
|
||||
request.WorkflowExecution,
|
||||
"wType",
|
||||
"testTaskQueue",
|
||||
payloads.EncodeString("input"),
|
||||
25*time.Second,
|
||||
20*time.Second,
|
||||
200*time.Second,
|
||||
"identity",
|
||||
)
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(
|
||||
&persistence.GetWorkflowExecutionResponse{State: workflow.TestCloneToProto(s.T().Context(), ms)},
|
||||
nil,
|
||||
)
|
||||
|
||||
mockClusterMetadata := cluster.NewMockMetadata(s.controller)
|
||||
mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo)
|
||||
mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes()
|
||||
s.mockShard.SetClusterMetadata(mockClusterMetadata)
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).
|
||||
Return(nil, serviceerror.NewUnavailable("source cluster unavailable"))
|
||||
|
||||
err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request)
|
||||
s.Require().ErrorAs(err, new(*serviceerror.Unavailable))
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_SkipsResendForRemovedNamespace() {
|
||||
s.config.EnableChildWorkflowResend = func() bool { return true }
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
capture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(capture)
|
||||
s.mockShard.SetMetricsHandler(metricsHandler)
|
||||
|
||||
namespaceID := namespace.ID(uuid.NewString())
|
||||
initialNamespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "removed-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestAlternativeClusterName,
|
||||
Clusters: cluster.TestAllClusterNames,
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
removedNamespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "removed-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestAlternativeClusterName,
|
||||
Clusters: []string{cluster.TestAlternativeClusterName},
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
resendChecked := make(chan struct{})
|
||||
gomock.InOrder(
|
||||
s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).Return(initialNamespaceEntry, nil),
|
||||
s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn(
|
||||
func(namespace.ID) (*namespace.Namespace, error) {
|
||||
close(resendChecked)
|
||||
return removedNamespaceEntry, nil
|
||||
},
|
||||
),
|
||||
)
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: namespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request)
|
||||
var notFound *serviceerror.NotFound
|
||||
s.ErrorAs(err, ¬Found)
|
||||
select {
|
||||
case <-resendChecked:
|
||||
case <-time.After(10 * time.Second):
|
||||
s.Fail("background child resend did not check namespace membership")
|
||||
}
|
||||
s.resendScheduler.InitiateShutdown()
|
||||
s.resendScheduler.WaitShutdown()
|
||||
metricSnapshot := capture.Snapshot()
|
||||
s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendAttempts.Name()], 1)
|
||||
s.Require().Empty(metricSnapshot[metrics.ChildWorkflowResendFailures.Name()])
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_SkipsApplyWhenActiveClusterChanges() {
|
||||
s.config.EnableChildWorkflowResend = func() bool { return true }
|
||||
|
||||
namespaceID := namespace.ID(uuid.NewString())
|
||||
initialNamespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "failover-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestCurrentClusterName,
|
||||
Clusters: cluster.TestAllClusterNames,
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
failedOverNamespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "failover-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestAlternativeClusterName,
|
||||
Clusters: cluster.TestAllClusterNames,
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
failedOver := make(chan struct{})
|
||||
s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn(
|
||||
func(namespace.ID) (*namespace.Namespace, error) {
|
||||
select {
|
||||
case <-failedOver:
|
||||
return failedOverNamespaceEntry, nil
|
||||
default:
|
||||
return initialNamespaceEntry, nil
|
||||
}
|
||||
},
|
||||
).AnyTimes()
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: namespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
|
||||
mockClusterMetadata := cluster.NewMockMetadata(s.controller)
|
||||
mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo)
|
||||
mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes()
|
||||
s.mockShard.SetClusterMetadata(mockClusterMetadata)
|
||||
|
||||
syncResponse := &adminservice.SyncWorkflowStateResponse{
|
||||
VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{},
|
||||
}
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), protomock.Eq(&adminservice.SyncWorkflowStateRequest{
|
||||
NamespaceId: request.NamespaceId,
|
||||
Execution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: request.WorkflowExecution.WorkflowId,
|
||||
RunId: request.WorkflowExecution.RunId,
|
||||
},
|
||||
TargetClusterId: int32(cluster.TestAlternativeClusterInitialFailoverVersion),
|
||||
ArchetypeId: chasm.WorkflowArchetypeID,
|
||||
})).DoAndReturn(
|
||||
func(context.Context, *adminservice.SyncWorkflowStateRequest, ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) {
|
||||
close(failedOver)
|
||||
return syncResponse, nil
|
||||
},
|
||||
)
|
||||
|
||||
err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request)
|
||||
var notFound *serviceerror.NotFound
|
||||
s.ErrorAs(err, ¬Found)
|
||||
select {
|
||||
case <-failedOver:
|
||||
case <-time.After(10 * time.Second):
|
||||
s.Fail("background child resend did not reach the source cluster")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildDeduped() {
|
||||
s.config.EnableChildWorkflowResend = func() bool { return true }
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
capture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(capture)
|
||||
s.mockShard.SetMetricsHandler(metricsHandler)
|
||||
|
||||
request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.NamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ResendChild: true,
|
||||
}
|
||||
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}).AnyTimes()
|
||||
|
||||
mockClusterMetadata := cluster.NewMockMetadata(s.controller)
|
||||
mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes()
|
||||
s.mockShard.SetClusterMetadata(mockClusterMetadata)
|
||||
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).
|
||||
DoAndReturn(func(ctx context.Context, _ *adminservice.SyncWorkflowStateRequest, _ ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) {
|
||||
close(entered)
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
close(finished)
|
||||
return nil, serviceerror.NewUnavailable("source cluster unavailable")
|
||||
}).Times(1)
|
||||
|
||||
ctx := metrics.AddMetricsContext(s.T().Context())
|
||||
err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(ctx, request)
|
||||
var notFound *serviceerror.NotFound
|
||||
s.ErrorAs(err, ¬Found)
|
||||
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(10 * time.Second):
|
||||
s.Fail("first child resend did not reach the source cluster")
|
||||
}
|
||||
|
||||
err = s.historyEngine.VerifyFirstWorkflowTaskScheduled(ctx, request)
|
||||
s.ErrorAs(err, ¬Found)
|
||||
|
||||
close(release)
|
||||
select {
|
||||
case <-finished:
|
||||
case <-time.After(10 * time.Second):
|
||||
s.Fail("background child resend did not finish")
|
||||
}
|
||||
s.resendScheduler.InitiateShutdown()
|
||||
s.resendScheduler.WaitShutdown()
|
||||
metricSnapshot := capture.Snapshot()
|
||||
s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendAttempts.Name()], 1)
|
||||
s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendSkipped.Name()], 1)
|
||||
s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendFailures.Name()], 1)
|
||||
s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendLatency.Name()], 1)
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_WorkflowNotExist() {
|
||||
|
||||
request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{
|
||||
@@ -2692,6 +3091,10 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParent()
|
||||
s.config.EnableAsyncParentWorkflowResend = func() bool { return false }
|
||||
s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true)
|
||||
capture := s.parentChildEventCapture
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
metricsCapture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(metricsCapture)
|
||||
s.mockShard.SetMetricsHandler(metricsHandler)
|
||||
|
||||
request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{
|
||||
NamespaceId: tests.ParentNamespaceID.String(),
|
||||
@@ -2836,6 +3239,70 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParent()
|
||||
s.Equal(wideevents.ParentChildOutcomeVerified, wideEventAttributes(sourceNotFoundRecord)["outcome"].AsString())
|
||||
sourceNotFoundDetails := wideEventDetails(sourceNotFoundRecord)
|
||||
s.Equal(util.ErrorType(serviceerror.NewNotFound("")), sourceNotFoundDetails["error_type"])
|
||||
metricSnapshot := metricsCapture.Snapshot()
|
||||
s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendAttempts.Name()], 2)
|
||||
s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendLatency.Name()], 2)
|
||||
s.Require().Empty(metricSnapshot[metrics.ParentWorkflowResendFailures.Name()])
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentInlinePreservesNamespaceNotFound() {
|
||||
s.config.EnableAsyncParentWorkflowResend = func() bool { return false }
|
||||
|
||||
namespaceID := namespace.ID(uuid.NewString())
|
||||
namespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "missing-parent-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestAlternativeClusterName,
|
||||
Clusters: cluster.TestAllClusterNames,
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
namespaceNotFoundErr := serviceerror.NewNamespaceNotFound(namespaceID.String())
|
||||
guardChecked := make(chan struct{})
|
||||
gomock.InOrder(
|
||||
s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).Return(namespaceEntry, nil),
|
||||
s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn(
|
||||
func(namespace.ID) (*namespace.Namespace, error) {
|
||||
close(guardChecked)
|
||||
return nil, namespaceNotFoundErr
|
||||
},
|
||||
),
|
||||
)
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Times(0)
|
||||
s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition(
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
).Times(0)
|
||||
|
||||
request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{
|
||||
NamespaceId: namespaceID.String(),
|
||||
ParentExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ChildExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: "child workflowId",
|
||||
RunId: "child runId",
|
||||
},
|
||||
ParentInitiatedId: 123,
|
||||
ParentInitiatedVersion: 100,
|
||||
ResendParent: true,
|
||||
}
|
||||
|
||||
_, err := s.historyEngine.VerifyChildExecutionCompletionRecorded(
|
||||
metrics.AddMetricsContext(s.T().Context()),
|
||||
request,
|
||||
)
|
||||
s.Require().Same(namespaceNotFoundErr, err)
|
||||
select {
|
||||
case <-guardChecked:
|
||||
default:
|
||||
s.Fail("inline parent resend did not reach the namespace guard")
|
||||
}
|
||||
}
|
||||
|
||||
// Async resend: the RPC returns the verification error immediately and the pull runs in the
|
||||
@@ -2907,7 +3374,7 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentAs
|
||||
|
||||
func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentLimited() {
|
||||
s.config.EnableAsyncParentWorkflowResend = func() bool { return true }
|
||||
s.config.ParentWorkflowResendMaxInFlight = func() int { return 0 }
|
||||
s.config.WorkflowResendHostMaxInFlight = func() int { return 0 }
|
||||
s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true)
|
||||
capture := s.parentChildEventCapture
|
||||
|
||||
@@ -2936,12 +3403,217 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentLi
|
||||
s.InDelta(0, details["max_in_flight"], 0)
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_SkipsResendForRemovedNamespace() {
|
||||
s.config.EnableAsyncParentWorkflowResend = func() bool { return true }
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
capture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(capture)
|
||||
s.mockShard.SetMetricsHandler(metricsHandler)
|
||||
|
||||
namespaceID := namespace.ID(uuid.NewString())
|
||||
initialNamespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "removed-parent-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestAlternativeClusterName,
|
||||
Clusters: cluster.TestAllClusterNames,
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
removedNamespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "removed-parent-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestAlternativeClusterName,
|
||||
Clusters: []string{cluster.TestAlternativeClusterName},
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
guardChecked := make(chan struct{})
|
||||
gomock.InOrder(
|
||||
s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).Return(initialNamespaceEntry, nil),
|
||||
s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn(
|
||||
func(namespace.ID) (*namespace.Namespace, error) {
|
||||
close(guardChecked)
|
||||
return removedNamespaceEntry, nil
|
||||
},
|
||||
),
|
||||
)
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Times(0)
|
||||
s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition(
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
).Times(0)
|
||||
|
||||
request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{
|
||||
NamespaceId: namespaceID.String(),
|
||||
ParentExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ChildExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: "child workflowId",
|
||||
RunId: "child runId",
|
||||
},
|
||||
ParentInitiatedId: 123,
|
||||
ParentInitiatedVersion: 100,
|
||||
ResendParent: true,
|
||||
}
|
||||
|
||||
_, err := s.historyEngine.VerifyChildExecutionCompletionRecorded(
|
||||
metrics.AddMetricsContext(s.T().Context()),
|
||||
request,
|
||||
)
|
||||
s.Require().ErrorAs(err, new(*serviceerror.NotFound))
|
||||
|
||||
select {
|
||||
case <-guardChecked:
|
||||
case <-time.After(10 * time.Second):
|
||||
s.Fail("background parent resend did not check namespace membership")
|
||||
}
|
||||
s.resendScheduler.InitiateShutdown()
|
||||
s.resendScheduler.WaitShutdown()
|
||||
metricSnapshot := capture.Snapshot()
|
||||
s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendAttempts.Name()], 1)
|
||||
s.Require().Empty(metricSnapshot[metrics.ParentWorkflowResendFailures.Name()])
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentAsyncSkipsApplyAfterFailover() {
|
||||
s.config.EnableAsyncParentWorkflowResend = func() bool { return true }
|
||||
|
||||
namespaceID := namespace.ID(uuid.NewString())
|
||||
initialNamespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "parent-resend-failover-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestCurrentClusterName,
|
||||
Clusters: cluster.TestAllClusterNames,
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
failedOverNamespaceEntry := namespace.NewGlobalNamespaceForTest(
|
||||
&persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "parent-resend-failover-namespace"},
|
||||
&persistencespb.NamespaceConfig{},
|
||||
&persistencespb.NamespaceReplicationConfig{
|
||||
ActiveClusterName: cluster.TestAlternativeClusterName,
|
||||
Clusters: cluster.TestAllClusterNames,
|
||||
},
|
||||
tests.Version,
|
||||
)
|
||||
failedOver := make(chan struct{})
|
||||
secondGuardLookup := make(chan struct{}, 1)
|
||||
s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn(
|
||||
func(namespace.ID) (*namespace.Namespace, error) {
|
||||
select {
|
||||
case <-failedOver:
|
||||
select {
|
||||
case secondGuardLookup <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return failedOverNamespaceEntry, nil
|
||||
default:
|
||||
return initialNamespaceEntry, nil
|
||||
}
|
||||
},
|
||||
).AnyTimes()
|
||||
|
||||
request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{
|
||||
NamespaceId: namespaceID.String(),
|
||||
ParentExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ChildExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: "child workflowId",
|
||||
RunId: "child runId",
|
||||
},
|
||||
ParentInitiatedId: 123,
|
||||
ParentInitiatedVersion: 100,
|
||||
ResendParent: true,
|
||||
}
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
|
||||
mockClusterMetadata := cluster.NewMockMetadata(s.controller)
|
||||
mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes()
|
||||
mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo)
|
||||
mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes()
|
||||
s.mockShard.SetClusterMetadata(mockClusterMetadata)
|
||||
|
||||
syncResponse := &adminservice.SyncWorkflowStateResponse{
|
||||
VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{},
|
||||
}
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), protomock.Eq(&adminservice.SyncWorkflowStateRequest{
|
||||
NamespaceId: request.NamespaceId,
|
||||
Execution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: request.ParentExecution.WorkflowId,
|
||||
RunId: request.ParentExecution.RunId,
|
||||
},
|
||||
TargetClusterId: int32(cluster.TestAlternativeClusterInitialFailoverVersion),
|
||||
ArchetypeId: chasm.WorkflowArchetypeID,
|
||||
})).DoAndReturn(
|
||||
func(context.Context, *adminservice.SyncWorkflowStateRequest, ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) {
|
||||
close(failedOver)
|
||||
return syncResponse, nil
|
||||
},
|
||||
)
|
||||
s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition(
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
gomock.Any(),
|
||||
).Times(0)
|
||||
|
||||
_, err := s.historyEngine.VerifyChildExecutionCompletionRecorded(metrics.AddMetricsContext(s.T().Context()), request)
|
||||
s.Require().ErrorAs(err, new(*serviceerror.NotFound))
|
||||
|
||||
select {
|
||||
case <-secondGuardLookup:
|
||||
case <-time.After(10 * time.Second):
|
||||
s.Fail("background parent resend did not recheck namespace state after SyncWorkflowState")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_HostAtCapacity() {
|
||||
s.config.EnableAsyncParentWorkflowResend = func() bool { return true }
|
||||
s.config.WorkflowResendHostMaxInFlight = func() int { return 0 }
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
capture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(capture)
|
||||
s.mockShard.SetMetricsHandler(metricsHandler)
|
||||
|
||||
request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{
|
||||
NamespaceId: tests.ParentNamespaceID.String(),
|
||||
ParentExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: tests.WorkflowID,
|
||||
RunId: tests.RunID,
|
||||
},
|
||||
ChildExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: "child workflowId",
|
||||
RunId: "child runId",
|
||||
},
|
||||
ResendParent: true,
|
||||
}
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{})
|
||||
|
||||
_, err := s.historyEngine.VerifyChildExecutionCompletionRecorded(metrics.AddMetricsContext(s.T().Context()), request)
|
||||
s.Require().ErrorAs(err, new(*serviceerror.NotFound))
|
||||
s.Require().Len(capture.Snapshot()[metrics.ParentWorkflowResendLimited.Name()], 1)
|
||||
}
|
||||
|
||||
// TestVerifyChildExecutionCompletionRecorded_ResendParentDeduped asserts that a second attempt for
|
||||
// the same parent does not start a concurrent resend while the first is still running.
|
||||
func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDeduped() {
|
||||
s.config.EnableAsyncParentWorkflowResend = func() bool { return true }
|
||||
s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true)
|
||||
capture := s.parentChildEventCapture
|
||||
metricsHandler := metricstest.NewCaptureHandler()
|
||||
metricsCapture := metricsHandler.StartCapture()
|
||||
defer metricsHandler.StopCapture(metricsCapture)
|
||||
s.mockShard.SetMetricsHandler(metricsHandler)
|
||||
|
||||
request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{
|
||||
NamespaceId: tests.ParentNamespaceID.String(),
|
||||
@@ -2977,9 +3649,13 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDe
|
||||
}
|
||||
}()
|
||||
s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).
|
||||
DoAndReturn(func(context.Context, *adminservice.SyncWorkflowStateRequest, ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) {
|
||||
DoAndReturn(func(ctx context.Context, _ *adminservice.SyncWorkflowStateRequest, _ ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return nil, serviceerror.NewUnavailable("source cluster unavailable")
|
||||
}).Times(1)
|
||||
|
||||
@@ -3015,6 +3691,13 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDe
|
||||
string(wideevents.ParentChildOutcomeDeduplicated),
|
||||
string(wideevents.ParentChildOutcomeFailed),
|
||||
}, parentChildOutcomes(capture))
|
||||
s.resendScheduler.InitiateShutdown()
|
||||
s.resendScheduler.WaitShutdown()
|
||||
metricSnapshot := metricsCapture.Snapshot()
|
||||
s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendAttempts.Name()], 1)
|
||||
s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendSkipped.Name()], 1)
|
||||
s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendFailures.Name()], 1)
|
||||
s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendLatency.Name()], 1)
|
||||
}
|
||||
|
||||
func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_WorkflowClosed() {
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"go.temporal.io/server/common/testing/testhooks"
|
||||
"go.temporal.io/server/common/worker_versioning"
|
||||
"go.temporal.io/server/service/history/api"
|
||||
"go.temporal.io/server/service/history/api/workflowresend"
|
||||
"go.temporal.io/server/service/history/circuitbreakerpool"
|
||||
"go.temporal.io/server/service/history/configs"
|
||||
"go.temporal.io/server/service/history/events"
|
||||
@@ -55,6 +56,7 @@ type (
|
||||
VersionMembershipCache worker_versioning.VersionMembershipAndReactivationStatusCache
|
||||
WorkerDeploymentClient workerdeployment.Client
|
||||
RoutingInfoCache worker_versioning.RoutingInfoCache
|
||||
WorkflowResendScheduler workflowresend.Scheduler `optional:"true"`
|
||||
}
|
||||
|
||||
historyEngineFactory struct {
|
||||
@@ -94,5 +96,6 @@ func (f *historyEngineFactory) CreateEngine(
|
||||
f.PersistenceRateLimiter,
|
||||
f.TestHooks,
|
||||
f.ChasmEngine,
|
||||
WithWorkflowResendScheduler(f.WorkflowResendScheduler),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ import (
|
||||
"go.temporal.io/server/common/testing/protorequire"
|
||||
"go.temporal.io/server/service/history/api"
|
||||
"go.temporal.io/server/service/history/api/getworkflowexecutionrawhistoryv2"
|
||||
"go.temporal.io/server/service/history/api/workflowresend"
|
||||
"go.temporal.io/server/service/history/configs"
|
||||
"go.temporal.io/server/service/history/consts"
|
||||
"go.temporal.io/server/service/history/events"
|
||||
@@ -84,6 +85,21 @@ const (
|
||||
esIndexName = ""
|
||||
)
|
||||
|
||||
func TestWithWorkflowResendScheduler(t *testing.T) {
|
||||
scheduler := workflowresend.NewBoundedWorkflowScheduler(
|
||||
func() int { return 1 },
|
||||
log.NewNoopLogger(),
|
||||
metrics.NoopMetricsHandler,
|
||||
)
|
||||
t.Cleanup(func() {
|
||||
scheduler.InitiateShutdown()
|
||||
scheduler.WaitShutdown()
|
||||
})
|
||||
|
||||
options := applyEngineOptions([]EngineOption{WithWorkflowResendScheduler(scheduler)})
|
||||
require.Same(t, scheduler, options.workflowResendScheduler)
|
||||
}
|
||||
|
||||
type (
|
||||
engineSuite struct {
|
||||
suite.Suite
|
||||
|
||||
@@ -157,6 +157,10 @@ type (
|
||||
parentWorkflowKey *definition.WorkflowKey
|
||||
}
|
||||
|
||||
startChildExecutionPostActionInfo struct {
|
||||
childWorkflowKey *definition.WorkflowKey
|
||||
}
|
||||
|
||||
workflowTaskPostActionInfo struct {
|
||||
workflowTaskScheduleToStartTimeout time.Duration
|
||||
taskqueue *taskqueuepb.TaskQueue
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
|
||||
const (
|
||||
recordChildCompletionVerificationFailedMsg = "Failed to verify child execution completion recorded"
|
||||
verifyFirstWorkflowTaskScheduledFailedMsg = "Failed to verify first workflow task scheduled"
|
||||
)
|
||||
|
||||
type (
|
||||
@@ -331,7 +332,7 @@ func (t *transferQueueStandbyTaskExecutor) processCloseExecution(
|
||||
taskTime := transferTask.GetVisibilityTime()
|
||||
localVerificationTime := taskTime.Add(t.config.MaxLocalParentWorkflowVerificationDuration())
|
||||
|
||||
resendParent := now.After(localVerificationTime) && mutableState.IsTransitionHistoryEnabled() && mutableState.CurrentVersionedTransition() != nil
|
||||
resendParent := now.After(localVerificationTime)
|
||||
|
||||
// Copy needed values from executionInfo before releasing mutable state
|
||||
parentNamespaceID := executionInfo.ParentNamespaceId
|
||||
@@ -344,6 +345,18 @@ func (t *transferQueueStandbyTaskExecutor) processCloseExecution(
|
||||
|
||||
// no need for mutable state anymore, release workflow lock
|
||||
release(nil)
|
||||
if resendParent {
|
||||
// Parent and child workflows may use different namespace transition-history settings.
|
||||
parentNamespaceEntry, err := t.registry.GetNamespaceByID(namespace.ID(parentNamespaceID))
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
resendParent = t.config.EnableTransitionHistory(parentNamespaceEntry.Name().String())
|
||||
case *serviceerror.NamespaceNotFound:
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
emitChildCompletionVerificationStarted(
|
||||
t.shardContext,
|
||||
@@ -519,6 +532,7 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution(
|
||||
childStartedWorkflowID := childWorkflowInfo.StartedWorkflowId
|
||||
childStartedRunID := childWorkflowInfo.StartedRunId
|
||||
childClock := childWorkflowInfo.Clock
|
||||
|
||||
// no need for mutable state anymore, release workflow lock
|
||||
release(nil)
|
||||
|
||||
@@ -532,7 +546,7 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution(
|
||||
}
|
||||
|
||||
if !childStarted {
|
||||
return &struct{}{}, nil
|
||||
return &startChildExecutionPostActionInfo{}, nil
|
||||
}
|
||||
|
||||
if childTargetNamespaceID == "" {
|
||||
@@ -545,6 +559,27 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution(
|
||||
}
|
||||
childTargetNamespaceID = targetNamespaceEntry.ID().String()
|
||||
}
|
||||
childWorkflowKey := definition.NewWorkflowKey(
|
||||
childTargetNamespaceID,
|
||||
childStartedWorkflowID,
|
||||
childStartedRunID,
|
||||
)
|
||||
resendTime := transferTask.GetVisibilityTime().Add(
|
||||
t.config.StandbyTaskMissingEventsResendDelay(transferTask.GetType()),
|
||||
)
|
||||
resendChild := t.getCurrentTime().After(resendTime)
|
||||
if resendChild {
|
||||
// Parent and child workflows may use different namespace transition-history settings.
|
||||
childNamespaceEntry, err := t.registry.GetNamespaceByID(namespace.ID(childTargetNamespaceID))
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
resendChild = t.config.EnableTransitionHistory(childNamespaceEntry.Name().String())
|
||||
case *serviceerror.NamespaceNotFound:
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = t.historyRawClient.VerifyFirstWorkflowTaskScheduled(ctx, &historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: childTargetNamespaceID,
|
||||
@@ -552,7 +587,8 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution(
|
||||
WorkflowId: childStartedWorkflowID,
|
||||
RunId: childStartedRunID,
|
||||
},
|
||||
Clock: childClock,
|
||||
Clock: childClock,
|
||||
ResendChild: resendChild,
|
||||
})
|
||||
switch err.(type) {
|
||||
case nil, *serviceerror.NamespaceNotFound, *serviceerror.Unimplemented:
|
||||
@@ -561,13 +597,15 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution(
|
||||
case *serviceerror.NotFound, *serviceerror.WorkflowNotReady:
|
||||
// Case 2: Target workflow is not in the desired state.
|
||||
// Return a non-nil pointer as postActionInfo here to indicate that verification is not done yet.
|
||||
return &struct{}{}, nil
|
||||
return &startChildExecutionPostActionInfo{
|
||||
childWorkflowKey: &childWorkflowKey,
|
||||
}, nil
|
||||
default:
|
||||
// Case 3: Verification itself failed.
|
||||
// NOTE: Wrapping the error as a verification error to prevent mutable state from being cleared and reloaded upon retry,
|
||||
// which is unnecessary as the error is in the target workflow, not this workflow.
|
||||
return nil, &verificationErr{
|
||||
msg: recordChildCompletionVerificationFailedMsg,
|
||||
msg: verifyFirstWorkflowTaskScheduledFailedMsg,
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
@@ -582,7 +620,7 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution(
|
||||
transferTask,
|
||||
t.getCurrentTime,
|
||||
t.config.StandbyTaskMissingEventsDiscardDelay(transferTask.GetType()),
|
||||
t.checkExecutionStillExistsOnSourceBeforeDiscard,
|
||||
t.checkStartChildExecutionStillExistsOnSourceBeforeDiscard,
|
||||
),
|
||||
)
|
||||
}
|
||||
@@ -729,6 +767,35 @@ func (t *transferQueueStandbyTaskExecutor) checkExecutionStillExistsOnSourceBefo
|
||||
return standbyTransferTaskPostActionTaskDiscarded(ctx, taskInfo, postActionInfo, logger)
|
||||
}
|
||||
|
||||
func (t *transferQueueStandbyTaskExecutor) checkStartChildExecutionStillExistsOnSourceBeforeDiscard(
|
||||
ctx context.Context,
|
||||
taskInfo tasks.Task,
|
||||
postActionInfo any,
|
||||
logger log.Logger,
|
||||
) error {
|
||||
if postActionInfo == nil {
|
||||
return nil
|
||||
}
|
||||
startChildInfo, ok := postActionInfo.(*startChildExecutionPostActionInfo)
|
||||
if !ok || startChildInfo.childWorkflowKey == nil {
|
||||
return t.checkExecutionStillExistsOnSourceBeforeDiscard(ctx, taskInfo, postActionInfo, logger)
|
||||
}
|
||||
|
||||
if !executionExistsOnSource(
|
||||
ctx,
|
||||
*startChildInfo.childWorkflowKey,
|
||||
chasm.WorkflowArchetypeID,
|
||||
logger,
|
||||
t.clusterName,
|
||||
t.clientBean,
|
||||
t.shardContext.GetNamespaceRegistry(),
|
||||
t.shardContext.ChasmRegistry(),
|
||||
) {
|
||||
return standbyTransferTaskPostActionTaskDiscarded(ctx, taskInfo, nil, logger)
|
||||
}
|
||||
return standbyTransferTaskPostActionTaskDiscarded(ctx, taskInfo, postActionInfo, logger)
|
||||
}
|
||||
|
||||
func (t *transferQueueStandbyTaskExecutor) checkParentWorkflowStillExistOnSourceBeforeDiscard(
|
||||
ctx context.Context,
|
||||
taskInfo tasks.Task,
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
persistencespb "go.temporal.io/server/api/persistence/v1"
|
||||
workflowspb "go.temporal.io/server/api/workflow/v1"
|
||||
"go.temporal.io/server/chasm"
|
||||
chasmworkflow "go.temporal.io/server/chasm/lib/workflow"
|
||||
"go.temporal.io/server/client"
|
||||
"go.temporal.io/server/common/archiver"
|
||||
"go.temporal.io/server/common/archiver/provider"
|
||||
@@ -143,6 +144,8 @@ func (s *transferQueueStandbyTaskExecutorSuite) SetupTest() {
|
||||
err := workflow.RegisterStateMachine(reg)
|
||||
s.NoError(err)
|
||||
s.mockShard.SetStateMachineRegistry(reg)
|
||||
err = s.mockShard.ChasmRegistry().Register(chasmworkflow.NewLibrary(chasmworkflow.NewRegistry()))
|
||||
s.NoError(err)
|
||||
|
||||
s.mockShard.SetEventsCacheForTesting(events.NewHostLevelEventsCache(
|
||||
s.mockShard.GetExecutionManager(),
|
||||
@@ -942,6 +945,13 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCloseExecution() {
|
||||
ignoredAttributes := wideEventAttributes(records[3])
|
||||
s.Equal(string(wideevents.ReplicationApplied), ignoredAttributes["phase"].AsString())
|
||||
s.Equal(wideevents.ParentChildOutcomeVerified, ignoredAttributes["outcome"].AsString())
|
||||
|
||||
persistenceMutableState.ExecutionInfo.TransitionHistory = nil
|
||||
s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler)
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil)
|
||||
s.mockHistoryClient.EXPECT().VerifyChildExecutionCompletionRecorded(gomock.Any(), expectedVerificationWithResendParentRequest).Return(nil, nil)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.NoError(resp.ExecutionErr)
|
||||
}
|
||||
|
||||
func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCancelExecution_Pending() {
|
||||
@@ -1241,7 +1251,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P
|
||||
event = addWorkflowTaskCompletedEvent(&s.Suite, mutableState, wt.ScheduledEventID, wt.StartedEventID, "some random identity")
|
||||
|
||||
taskID := s.mustGenerateTaskID()
|
||||
event, _ = addStartChildWorkflowExecutionInitiatedEvent(mutableState, event.GetEventId(),
|
||||
event, childInfo := addStartChildWorkflowExecutionInitiatedEvent(mutableState, event.GetEventId(),
|
||||
tests.ChildNamespace, tests.ChildNamespaceID, childWorkflowID, childWorkflowType, childTaskQueueName, nil, 1*time.Second, 1*time.Second, 1*time.Second, enumspb.PARENT_CLOSE_POLICY_ABANDON)
|
||||
|
||||
now := time.Now().UTC()
|
||||
@@ -1264,7 +1274,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P
|
||||
resp := s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.Equal(consts.ErrTaskRetry, resp.ExecutionErr)
|
||||
|
||||
s.mockShard.SetCurrentTime(s.clusterName, now.Add(s.fetchHistoryDuration))
|
||||
s.mockShard.SetCurrentTime(s.clusterName, now.Add(time.Minute))
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.Equal(consts.ErrTaskRetry, resp.ExecutionErr)
|
||||
|
||||
@@ -1277,41 +1287,99 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P
|
||||
persistenceMutableState = s.createPersistenceMutableState(mutableState, event.GetEventId(), event.GetVersion())
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil)
|
||||
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, nil)
|
||||
expectedVerificationRequest := protomock.Eq(&historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.ChildNamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: childWorkflowID,
|
||||
RunId: childRunID,
|
||||
},
|
||||
Clock: childInfo.Clock,
|
||||
})
|
||||
expectedVerificationWithResendChildRequest := protomock.Eq(&historyservice.VerifyFirstWorkflowTaskScheduledRequest{
|
||||
NamespaceId: tests.ChildNamespaceID.String(),
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: childWorkflowID,
|
||||
RunId: childRunID,
|
||||
},
|
||||
Clock: childInfo.Clock,
|
||||
ResendChild: true,
|
||||
})
|
||||
|
||||
s.mockShard.SetCurrentTime(s.clusterName, now)
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, nil)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.NoError(resp.ExecutionErr)
|
||||
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, consts.ErrWorkflowNotReady)
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, consts.ErrWorkflowNotReady)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.Equal(consts.ErrTaskRetry, resp.ExecutionErr)
|
||||
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, consts.ErrWorkflowExecutionNotFound)
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, consts.ErrWorkflowExecutionNotFound)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.Equal(consts.ErrTaskRetry, resp.ExecutionErr)
|
||||
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.Unimplemented{})
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, &serviceerror.Unimplemented{})
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.NoError(resp.ExecutionErr)
|
||||
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, consts.ErrResourceExhaustedBusyWorkflow)
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, consts.ErrResourceExhaustedBusyWorkflow)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
var verificationErr *verificationErr
|
||||
s.ErrorAs(resp.ExecutionErr, &verificationErr)
|
||||
var resourceExhaustedErr *serviceerror.ResourceExhausted
|
||||
s.ErrorAs(resp.ExecutionErr, &resourceExhaustedErr)
|
||||
|
||||
s.mockShard.SetCurrentTime(s.clusterName, now.Add(s.fetchHistoryDuration))
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, consts.ErrWorkflowNotReady)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.Equal(consts.ErrTaskRetry, resp.ExecutionErr)
|
||||
|
||||
childExecution := &commonpb.WorkflowExecution{
|
||||
WorkflowId: childWorkflowID,
|
||||
RunId: childRunID,
|
||||
}
|
||||
expectedDescribeChildMutableStateRequest := protomock.Eq(&adminservice.DescribeMutableStateRequest{
|
||||
Namespace: tests.ChildNamespace.String(),
|
||||
Execution: childExecution,
|
||||
Archetype: chasm.WorkflowArchetype,
|
||||
SkipForceReload: true,
|
||||
})
|
||||
s.clientBean.EXPECT().GetRemoteAdminClient(
|
||||
tests.GlobalChildNamespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: childWorkflowID}),
|
||||
).Return(s.mockRemoteAdminClient, nil).AnyTimes()
|
||||
|
||||
s.mockShard.SetCurrentTime(s.clusterName, now.Add(s.discardDuration))
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.WorkflowNotReady{})
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, &serviceerror.WorkflowNotReady{})
|
||||
s.mockRemoteAdminClient.EXPECT().DescribeMutableState(gomock.Any(), expectedDescribeChildMutableStateRequest).Return(nil, nil)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.Equal(consts.ErrTaskDiscarded, resp.ExecutionErr)
|
||||
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, &serviceerror.WorkflowNotReady{})
|
||||
s.mockRemoteAdminClient.EXPECT().DescribeMutableState(gomock.Any(), expectedDescribeChildMutableStateRequest).Return(nil, &serviceerror.NotFound{})
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.NoError(resp.ExecutionErr)
|
||||
|
||||
randomErr := errors.New("some random error")
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, randomErr)
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, randomErr)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.ErrorAs(resp.ExecutionErr, &verificationErr)
|
||||
s.Equal(randomErr, verificationErr.Unwrap())
|
||||
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, nil)
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, nil)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.NoError(resp.ExecutionErr)
|
||||
|
||||
persistenceMutableState.ExecutionInfo.TransitionHistory = nil
|
||||
s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler)
|
||||
s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil)
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, nil)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.NoError(resp.ExecutionErr)
|
||||
|
||||
s.mockShard.GetConfig().EnableTransitionHistory = func(namespaceName string) bool {
|
||||
return namespaceName != tests.ChildNamespace.String()
|
||||
}
|
||||
s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, nil)
|
||||
resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask))
|
||||
s.NoError(resp.ExecutionErr)
|
||||
|
||||
|
||||
@@ -76,13 +76,14 @@ type (
|
||||
parentTestVars *testvars.TestVars
|
||||
childTestVars *testvars.TestVars
|
||||
|
||||
activeClusterIndex int
|
||||
gates [2]*parentChildReplicationGate
|
||||
removeHooks []func()
|
||||
cleanups []func()
|
||||
delayedTasks map[parentChildReplicationLane]*parentChildReplicationTask
|
||||
metricCaptures [2]parentChildMetricCapture
|
||||
trace []string
|
||||
activeClusterIndex int
|
||||
gates [2]*parentChildReplicationGate
|
||||
removeHooks []func()
|
||||
cleanups []func()
|
||||
legacyReplicationCleanups []func()
|
||||
delayedTasks map[parentChildReplicationLane]*parentChildReplicationTask
|
||||
metricCaptures [2]parentChildMetricCapture
|
||||
trace []string
|
||||
}
|
||||
|
||||
parentChildMetricCapture struct {
|
||||
@@ -248,17 +249,32 @@ func useLegacyHistoryReplication() parentChildScenarioStep {
|
||||
name: "use legacy history replication for this scenario",
|
||||
run: func(_ context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
for _, cluster := range runtime.suite.clusters {
|
||||
runtime.cleanups = append(runtime.cleanups, cluster.OverrideDynamicConfig(
|
||||
cleanup := cluster.OverrideDynamicConfig(
|
||||
runtime.suite.T(),
|
||||
dynamicconfig.EnableTransitionHistory,
|
||||
false,
|
||||
))
|
||||
)
|
||||
runtime.cleanups = append(runtime.cleanups, cleanup)
|
||||
runtime.legacyReplicationCleanups = append(runtime.legacyReplicationCleanups, cleanup)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func useTransitionHistory() parentChildScenarioStep {
|
||||
return parentChildScenarioStep{
|
||||
name: "use transition history for this scenario",
|
||||
run: func(_ context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
for index := len(runtime.legacyReplicationCleanups) - 1; index >= 0; index-- {
|
||||
runtime.legacyReplicationCleanups[index]()
|
||||
}
|
||||
runtime.legacyReplicationCleanups = nil
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func setLocalParentVerificationGrace(
|
||||
cluster parentChildCluster,
|
||||
duration time.Duration,
|
||||
@@ -326,6 +342,49 @@ func setStandbyTaskDiscardDelay(
|
||||
}
|
||||
}
|
||||
|
||||
func enableChildWorkflowResend(cluster parentChildCluster) parentChildScenarioStep {
|
||||
return parentChildScenarioStep{
|
||||
name: fmt.Sprintf("enable child workflow resend on %s", cluster),
|
||||
run: func(_ context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
clusterIndex := int(cluster)
|
||||
if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) {
|
||||
return fmt.Errorf("unknown parent-child cluster %d", cluster)
|
||||
}
|
||||
runtime.cleanups = append(runtime.cleanups, runtime.suite.clusters[clusterIndex].OverrideDynamicConfig(
|
||||
runtime.suite.T(),
|
||||
dynamicconfig.EnableChildWorkflowResend,
|
||||
true,
|
||||
))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func setStandbyTaskResendDelay(
|
||||
cluster parentChildCluster,
|
||||
taskType enumsspb.TaskType,
|
||||
duration time.Duration,
|
||||
) parentChildScenarioStep {
|
||||
return parentChildScenarioStep{
|
||||
name: fmt.Sprintf("set standby %s resend delay on %s to %s", taskType, cluster, duration),
|
||||
run: func(_ context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
clusterIndex := int(cluster)
|
||||
if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) {
|
||||
return fmt.Errorf("unknown parent-child cluster %d", cluster)
|
||||
}
|
||||
runtime.cleanups = append(runtime.cleanups, runtime.suite.clusters[clusterIndex].OverrideDynamicConfig(
|
||||
runtime.suite.T(),
|
||||
dynamicconfig.StandbyTaskMissingEventsResendDelay,
|
||||
[]dynamicconfig.ConstrainedValue{{
|
||||
Constraints: dynamicconfig.Constraints{TaskType: taskType},
|
||||
Value: duration,
|
||||
}},
|
||||
))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func startParentWorkflow() parentChildScenarioStep {
|
||||
return parentChildScenarioStep{
|
||||
name: "start parent workflow on the active cluster",
|
||||
@@ -364,6 +423,18 @@ func applyDelayedReplication(
|
||||
}
|
||||
}
|
||||
|
||||
func acknowledgeDelayedReplicationWithoutApplying(
|
||||
targetCluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
) parentChildScenarioStep {
|
||||
return parentChildScenarioStep{
|
||||
name: fmt.Sprintf("acknowledge delayed %s replication to %s without applying", workflow, targetCluster),
|
||||
run: func(_ context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
return runtime.acknowledgeDelayedReplicationWithoutApplying(targetCluster, workflow)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func acknowledgeReplicationTaskContainingEventWithoutApplying(
|
||||
targetCluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
@@ -372,6 +443,18 @@ func acknowledgeReplicationTaskContainingEventWithoutApplying(
|
||||
return replicationTaskStep(ackReplicationTaskWithoutApplying, targetCluster, workflow, eventType)
|
||||
}
|
||||
|
||||
func acknowledgeNextReplicationTaskWithoutApplying(
|
||||
targetCluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
) parentChildScenarioStep {
|
||||
return parentChildScenarioStep{
|
||||
name: fmt.Sprintf("acknowledge next %s replication to %s without applying", workflow, targetCluster),
|
||||
run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
return runtime.acknowledgeNextReplicationTaskWithoutApplying(ctx, targetCluster, workflow)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func replicationTaskStep(
|
||||
action parentChildReplicationTaskAction,
|
||||
targetCluster parentChildCluster,
|
||||
@@ -404,6 +487,15 @@ func completeChildWorkflowTask() parentChildScenarioStep {
|
||||
}
|
||||
}
|
||||
|
||||
func signalChildWorkflow() parentChildScenarioStep {
|
||||
return parentChildScenarioStep{
|
||||
name: "signal the child workflow on the active cluster",
|
||||
run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
return runtime.signalChildWorkflow(ctx)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func waitForWorkflowEventOnCluster(
|
||||
cluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
@@ -477,18 +569,6 @@ func currentWorkflowHasStatusOnCluster(
|
||||
}
|
||||
}
|
||||
|
||||
func workflowIsMissingOnCluster(
|
||||
cluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
) parentChildExpectation {
|
||||
return parentChildExpectation{
|
||||
name: fmt.Sprintf("%s is missing on %s", workflow, cluster),
|
||||
check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
return runtime.confirmWorkflowMissing(ctx, cluster, workflow)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func workflowHasEventOnCluster(
|
||||
cluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
@@ -651,6 +731,37 @@ func (r *parentChildScenarioRuntime) processReplicationThroughTaskContainingEven
|
||||
}
|
||||
}
|
||||
|
||||
func (r *parentChildScenarioRuntime) acknowledgeNextReplicationTaskWithoutApplying(
|
||||
ctx context.Context,
|
||||
targetCluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
) error {
|
||||
targetClusterIndex := int(targetCluster)
|
||||
if targetClusterIndex < 0 || targetClusterIndex >= len(r.gates) {
|
||||
return fmt.Errorf("unknown parent-child cluster %d", targetCluster)
|
||||
}
|
||||
workflowID, err := r.workflowID(workflow)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task, err := r.gates[targetClusterIndex].nextForWorkflow(ctx, workflowID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
events, err := decodeParentChildReplicationEvents(task.task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.tracef(
|
||||
" acknowledge task %d to cluster %d for %s without applying [%s]",
|
||||
task.task.GetSourceTaskId(),
|
||||
targetClusterIndex,
|
||||
workflow,
|
||||
formatParentChildReplicationTask(task.task, events),
|
||||
)
|
||||
return task.acknowledgeWithoutApplying()
|
||||
}
|
||||
|
||||
func (r *parentChildScenarioRuntime) applyDelayedReplication(
|
||||
targetCluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
@@ -688,6 +799,39 @@ func (r *parentChildScenarioRuntime) applyDelayedReplication(
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *parentChildScenarioRuntime) acknowledgeDelayedReplicationWithoutApplying(
|
||||
targetCluster parentChildCluster,
|
||||
workflow parentChildWorkflow,
|
||||
) error {
|
||||
targetClusterIndex := int(targetCluster)
|
||||
if targetClusterIndex < 0 || targetClusterIndex >= len(r.gates) {
|
||||
return fmt.Errorf("unknown parent-child cluster %d", targetCluster)
|
||||
}
|
||||
if _, err := r.workflowID(workflow); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lane := parentChildReplicationLane{targetClusterIndex: targetClusterIndex, workflow: workflow}
|
||||
task, delayed := r.delayedTasks[lane]
|
||||
if !delayed {
|
||||
return fmt.Errorf("no delayed %s replication task to %s", workflow, targetCluster)
|
||||
}
|
||||
events, err := decodeParentChildReplicationEvents(task.task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.tracef(
|
||||
" acknowledge delayed task %d to cluster %d for %s without applying [%s]",
|
||||
task.task.GetSourceTaskId(),
|
||||
targetClusterIndex,
|
||||
workflow,
|
||||
formatParentChildReplicationTask(task.task, events),
|
||||
)
|
||||
delete(r.delayedTasks, lane)
|
||||
return task.acknowledgeWithoutApplying()
|
||||
}
|
||||
|
||||
func (r *parentChildScenarioRuntime) completeParentWorkflowTaskWithStartChildCommand(ctx context.Context) error {
|
||||
if r.parentRunID == "" {
|
||||
return errors.New("parent workflow is not started")
|
||||
@@ -751,6 +895,23 @@ func (r *parentChildScenarioRuntime) completeChildWorkflowTask(ctx context.Conte
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *parentChildScenarioRuntime) signalChildWorkflow(ctx context.Context) error {
|
||||
if r.childRunID == "" {
|
||||
return errors.New("child workflow is not started")
|
||||
}
|
||||
_, err := r.activeCluster().FrontendClient().SignalWorkflowExecution(ctx, &workflowservice.SignalWorkflowExecutionRequest{
|
||||
Namespace: r.namespace,
|
||||
WorkflowExecution: &commonpb.WorkflowExecution{
|
||||
WorkflowId: r.childID,
|
||||
RunId: r.childRunID,
|
||||
},
|
||||
SignalName: "initialize-transition-history",
|
||||
Identity: r.childTestVars.WorkerIdentity(),
|
||||
RequestId: uuid.NewString(),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *parentChildScenarioRuntime) workflowHistoryOnCluster(
|
||||
ctx context.Context,
|
||||
cluster parentChildCluster,
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
historypb "go.temporal.io/api/history/v1"
|
||||
"go.temporal.io/api/serviceerror"
|
||||
enumsspb "go.temporal.io/server/api/enums/v1"
|
||||
"go.temporal.io/server/common"
|
||||
"go.temporal.io/server/common/metrics"
|
||||
"go.temporal.io/server/tests/testcore"
|
||||
)
|
||||
@@ -192,25 +191,33 @@ func (s *parentChildXDCTestSuite) TestReproOrphanedChildAfterForceFailover() {
|
||||
})
|
||||
}
|
||||
|
||||
// TestStandbyVerifiesMissingChild covers the cross-shard ordering where the parent update
|
||||
// TestStandbyResendsMissingChild covers the cross-shard ordering where the parent update
|
||||
// identifying a started child reaches the passive while the child's WorkflowExecutionStarted
|
||||
// task remains delayed.
|
||||
//
|
||||
// | parent on target | child on target | active | outcome
|
||||
// ---------------------------+---------------------------------+-----------------+----------------+---------------------------------------------
|
||||
// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix
|
||||
// child start is delayed | unchanged | does not exist | initial active | child start remains unapplied
|
||||
// parent child-start arrives | ChildWorkflowExecutionStarted | does not exist | initial active | standby verifies the child's first WFT
|
||||
// verification fails | child-start relationship exists | does not exist | initial active | VerifyFirstWorkflowTaskScheduled: NotFound
|
||||
// discard window expires | unchanged | does not exist | initial active | standby StartChild task is discarded
|
||||
// | parent on target | child on target | active | outcome
|
||||
// ---------------------------+---------------------------------+-----------------------+----------------+---------------------------------------------
|
||||
// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix
|
||||
// child start is delayed | unchanged | does not exist | initial active | child start remains unapplied
|
||||
// parent child-start arrives | ChildWorkflowExecutionStarted | does not exist | initial active | standby verifies the child's first WFT
|
||||
// verification fails | child-start relationship exists | does not exist | initial active | VerifyFirstWorkflowTaskScheduled: NotFound
|
||||
// child is resent | unchanged | RUNNING, WFT scheduled | initial active | child state is restored from the source
|
||||
//
|
||||
// Event checkpoints select an entire replication task, not an individual event. The delayed child
|
||||
// start task intentionally remains unapplied while the standby StartChild task is allowed to expire.
|
||||
func (s *parentChildXDCTestSuite) TestStandbyVerifiesMissingChild() {
|
||||
// start task intentionally remains unapplied while child state is restored through state sync.
|
||||
func (s *parentChildXDCTestSuite) TestStandbyResendsMissingChild() {
|
||||
s.runParentChildScenario(parentChildScenario{
|
||||
steps: []parentChildScenarioStep{
|
||||
// Track source time without the production standby lag so task expiration is observable quickly.
|
||||
// Track source time without the production standby lag so the task becomes resend-eligible quickly.
|
||||
setStandbyClusterDelay(initialStandbyCluster, 0),
|
||||
// Enable the child resend path on the passive cluster.
|
||||
enableChildWorkflowResend(initialStandbyCluster),
|
||||
// Skip the normal resend delay so the pending standby StartChild task requests a state sync.
|
||||
setStandbyTaskResendDelay(
|
||||
initialStandbyCluster,
|
||||
enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION,
|
||||
0,
|
||||
),
|
||||
// Create the parent and its first workflow task on the initial active cluster.
|
||||
startParentWorkflow(),
|
||||
// Establish the parent on the passive before replicating its child relationship.
|
||||
@@ -227,36 +234,42 @@ func (s *parentChildXDCTestSuite) TestStandbyVerifiesMissingChild() {
|
||||
childWorkflow,
|
||||
enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED,
|
||||
),
|
||||
// Confirm that the ordinary child replication task remains unapplied before verification.
|
||||
confirmWorkflowIsMissingOnCluster(initialStandbyCluster, childWorkflow),
|
||||
// Apply the parent's child-start record, triggering verification of the missing child.
|
||||
applyReplicationThroughTaskContainingEvent(
|
||||
initialStandbyCluster,
|
||||
parentWorkflow,
|
||||
enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED,
|
||||
),
|
||||
// Observe the exact missing-child error before shortening the task's discard window.
|
||||
// Observe the original missing-child error returned while the resend runs in the background.
|
||||
waitForHistoryVerificationFailureOnCluster(
|
||||
initialStandbyCluster,
|
||||
historyClientVerifyFirstWorkflowTask,
|
||||
&serviceerror.NotFound{},
|
||||
),
|
||||
// Expire only the pending standby StartChild task; its next attempt should return ErrTaskDiscarded.
|
||||
setStandbyTaskDiscardDelay(
|
||||
initialStandbyCluster,
|
||||
enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION,
|
||||
0,
|
||||
),
|
||||
},
|
||||
expectations: []parentChildExpectation{
|
||||
workflowIsMissingOnCluster(initialStandbyCluster, childWorkflow),
|
||||
taskWasDiscardedOnCluster(
|
||||
{
|
||||
name: "child workflow resend is attempted on the initial standby cluster",
|
||||
check: func(_ context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
return runtime.requireCapturedMetric(
|
||||
initialStandbyCluster,
|
||||
metrics.ChildWorkflowResendAttempts.Name(),
|
||||
nil,
|
||||
)
|
||||
},
|
||||
},
|
||||
workflowHasEventOnCluster(
|
||||
initialStandbyCluster,
|
||||
metrics.TaskTypeTransferStandbyTaskStartChildExecution,
|
||||
childWorkflow,
|
||||
enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED,
|
||||
),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestStandbyVerifiesChildWithoutFirstWorkflowTask covers the cross-shard ordering where the
|
||||
// TestStandbyResendsChildWithoutFirstWorkflowTask covers the cross-shard ordering where the
|
||||
// child's WorkflowExecutionStarted reaches the passive, its first WorkflowTaskScheduled remains
|
||||
// delayed, and the parent update identifying the started child then arrives.
|
||||
//
|
||||
@@ -267,18 +280,25 @@ func (s *parentChildXDCTestSuite) TestStandbyVerifiesMissingChild() {
|
||||
// child first WFT is delayed | unchanged | unchanged | initial active | WorkflowTaskScheduled remains unapplied
|
||||
// parent child-start arrives | ChildWorkflowExecutionStarted | unchanged | initial active | standby verifies the child's first WFT
|
||||
// verification fails | child-start relationship exists | next event ID 2, no scheduled ID | initial active | VerifyFirstWorkflowTaskScheduled: WorkflowNotReady
|
||||
// discard window expires | unchanged | unchanged | initial active | standby StartChild task is discarded
|
||||
// child is resent | unchanged | RUNNING, WFT scheduled | initial active | child state is restored from the source
|
||||
//
|
||||
// This scenario uses legacy history replication because it fixes each transaction's event range.
|
||||
// The gate can therefore apply WorkflowExecutionStarted while keeping the separate
|
||||
// WorkflowTaskScheduled task delayed through the assertions.
|
||||
func (s *parentChildXDCTestSuite) TestStandbyVerifiesChildWithoutFirstWorkflowTask() {
|
||||
// The replication gate applies WorkflowExecutionStarted while keeping the separate
|
||||
// WorkflowTaskScheduled update delayed, so recovery must come from state sync.
|
||||
func (s *parentChildXDCTestSuite) TestStandbyResendsChildWithoutFirstWorkflowTask() {
|
||||
s.runParentChildScenario(parentChildScenario{
|
||||
steps: []parentChildScenarioStep{
|
||||
// Keep child Started and its first WFT in separate event-range replication tasks.
|
||||
useLegacyHistoryReplication(),
|
||||
// Track source time without the production standby lag so task expiration is observable quickly.
|
||||
// Track source time without the production standby lag so the task becomes resend-eligible quickly.
|
||||
setStandbyClusterDelay(initialStandbyCluster, 0),
|
||||
// Enable the child resend path on the passive cluster.
|
||||
enableChildWorkflowResend(initialStandbyCluster),
|
||||
// Skip the normal resend delay so the pending standby StartChild task requests a state sync.
|
||||
setStandbyTaskResendDelay(
|
||||
initialStandbyCluster,
|
||||
enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION,
|
||||
0,
|
||||
),
|
||||
// Create the parent and its first workflow task on the initial active cluster.
|
||||
startParentWorkflow(),
|
||||
// Establish the parent on the passive before replicating its child relationship.
|
||||
@@ -301,51 +321,42 @@ func (s *parentChildXDCTestSuite) TestStandbyVerifiesChildWithoutFirstWorkflowTa
|
||||
childWorkflow,
|
||||
enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED,
|
||||
),
|
||||
// Drop the ordinary WFT replication task so it cannot heal the passive child.
|
||||
acknowledgeDelayedReplicationWithoutApplying(initialStandbyCluster, childWorkflow),
|
||||
// State sync uses transition history after the partial child state has been reproduced.
|
||||
useTransitionHistory(),
|
||||
// Initialize transition history on the source child while retaining its scheduled WFT.
|
||||
signalChildWorkflow(),
|
||||
// Do not let the transition update repair the passive through ordinary replication.
|
||||
acknowledgeNextReplicationTaskWithoutApplying(initialStandbyCluster, childWorkflow),
|
||||
// Apply the parent's child-start record, triggering verification of that partial child.
|
||||
applyReplicationThroughTaskContainingEvent(
|
||||
initialStandbyCluster,
|
||||
parentWorkflow,
|
||||
enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED,
|
||||
),
|
||||
// Observe the exact not-ready error before shortening the task's discard window.
|
||||
// Observe the original not-ready error returned while the resend runs in the background.
|
||||
waitForHistoryVerificationFailureOnCluster(
|
||||
initialStandbyCluster,
|
||||
historyClientVerifyFirstWorkflowTask,
|
||||
&serviceerror.WorkflowNotReady{},
|
||||
),
|
||||
// Expire only the pending standby StartChild task; its next attempt should return ErrTaskDiscarded.
|
||||
setStandbyTaskDiscardDelay(
|
||||
initialStandbyCluster,
|
||||
enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION,
|
||||
0,
|
||||
),
|
||||
},
|
||||
expectations: []parentChildExpectation{
|
||||
{
|
||||
name: "child exists without its first workflow task on the initial standby cluster",
|
||||
check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
mutableState, err := runtime.workflowMutableState(ctx, initialStandbyCluster, childWorkflow)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state := mutableState.GetExecutionState().GetState(); state != enumsspb.WORKFLOW_EXECUTION_STATE_CREATED {
|
||||
return fmt.Errorf("child state is %s, want %s", state, enumsspb.WORKFLOW_EXECUTION_STATE_CREATED)
|
||||
}
|
||||
if status := mutableState.GetExecutionState().GetStatus(); status != enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING {
|
||||
return fmt.Errorf("child status is %s, want %s", status, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING)
|
||||
}
|
||||
if scheduledEventID := mutableState.GetExecutionInfo().GetWorkflowTaskScheduledEventId(); scheduledEventID != common.EmptyEventID {
|
||||
return fmt.Errorf("child workflow task scheduled event ID is %d, want %d", scheduledEventID, common.EmptyEventID)
|
||||
}
|
||||
if nextEventID := mutableState.GetNextEventId(); nextEventID != common.FirstEventID+1 {
|
||||
return fmt.Errorf("child next event ID is %d, want %d", nextEventID, common.FirstEventID+1)
|
||||
}
|
||||
return nil
|
||||
name: "child workflow resend is attempted on the initial standby cluster",
|
||||
check: func(_ context.Context, runtime *parentChildScenarioRuntime) error {
|
||||
return runtime.requireCapturedMetric(
|
||||
initialStandbyCluster,
|
||||
metrics.ChildWorkflowResendAttempts.Name(),
|
||||
nil,
|
||||
)
|
||||
},
|
||||
},
|
||||
taskWasDiscardedOnCluster(
|
||||
workflowHasEventOnCluster(
|
||||
initialStandbyCluster,
|
||||
metrics.TaskTypeTransferStandbyTaskStartChildExecution,
|
||||
childWorkflow,
|
||||
enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED,
|
||||
),
|
||||
},
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user