mirror of
https://github.com/temporalio/temporal.git
synced 2026-08-30 18:41:49 -07:00
Fix pinned workflow trampolining via revision-based signal suppression (#9895)
## Summary
- Matching now sends the real `targetDeploymentRevisionNumber` for
pinned workflows instead of hardcoded 0
- Added `revision_number` field to `LastNotifiedTargetVersion`
(server-internal) and `DeclinedTargetVersionUpgrade` (public API)
- Case 4 in the target version change switch now uses
`targetRevisionNumber <= declined.RevisionNumber` instead of version
string comparison, preventing trampolining when stale matching
partitions send outdated target versions
## Problem
When a pinned workflow CaNs and declines a target version upgrade, a
stale matching partition can send an older target version. The old case
4 compared version strings (`declined.buildId == target.buildId`), which
didn't match the stale version — causing the workflow to re-signal, CaN
again, and trampoline indefinitely between stale and up-to-date
partitions.
## Test plan
- [x] `TestStalePartition_RevisionSuppressesTrampolining` — integration
test that simulates a stale partition via `rollbackTaskQueueToVersion`
and verifies:
- Stale partition (revision 0) is suppressed after declining at a higher
revision
- Genuinely new version (v4 at higher revision) correctly fires the
signal
- [x] Verified test **fails** when matching + history changes are
reverted (matching sends 0, old string comparison)
## API repo dependency
- api: `temporalio/api@trampolining-rev-number`
- api-go: `temporalio/api-go@trampolining-rev-number`
🤖 Generated with [Claude Code](https://claude.com/claude-code)
<!-- CURSOR_SUMMARY -->
---
> [!NOTE]
> **Medium Risk**
> Touches pinned-workflow versioning logic across matching, history, and
persistence; a bug here could change when workflows are signaled to
continue-as-new or upgrade, affecting routing behavior.
>
> **Overview**
> Prevents pinned workflows from repeatedly continue-as-new
“trampolining” when task queue partitions have stale routing data by
**tracking and comparing target routing-config revision numbers**.
>
> Matching now propagates the real `targetDeploymentRevisionNumber` for
pinned workflow tasks, and history threads this through
`AddWorkflowTaskStartedEvent` to persist
`LastNotifiedTargetVersion.revision_number` and carry it into
`DeclinedTargetVersionUpgrade` on continue-as-new. The pinned
target-change decision in `workflow_task_state_machine` switches case-4
suppression from deployment version string equality to
`targetRevisionNumber <= declined.RevisionNumber`, and adds an
integration test (`TestStalePartition_RevisionSuppressesTrampolining`)
covering stale vs fresh partition behavior.
>
> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit
4114662b39. Bugbot is set up for automated
code reviews on this repo. Configure
[here](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- /CURSOR_SUMMARY -->
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1233,8 +1233,12 @@ func (x *TimeSkippingInfo) GetAccumulatedSkippedDuration() *durationpb.Duration
|
||||
type LastNotifiedTargetVersion struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
DeploymentVersion *v18.WorkerDeploymentVersion `protobuf:"bytes,1,opt,name=deployment_version,json=deploymentVersion,proto3" json:"deployment_version,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// Revision number of the task queue routing config at the time the
|
||||
// notification was sent. Carried forward to DeclinedTargetVersionUpgrade
|
||||
// at continue-as-new time.
|
||||
RevisionNumber int64 `protobuf:"varint,2,opt,name=revision_number,json=revisionNumber,proto3" json:"revision_number,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *LastNotifiedTargetVersion) Reset() {
|
||||
@@ -1274,6 +1278,13 @@ func (x *LastNotifiedTargetVersion) GetDeploymentVersion() *v18.WorkerDeployment
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *LastNotifiedTargetVersion) GetRevisionNumber() int64 {
|
||||
if x != nil {
|
||||
return x.RevisionNumber
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type ExecutionStats struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
HistorySize int64 `protobuf:"varint,1,opt,name=history_size,json=historySize,proto3" json:"history_size,omitempty"`
|
||||
@@ -4949,9 +4960,10 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" +
|
||||
"\x1alast_workflow_task_failureJ\x04\b\b\x10\tJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10J\x04\b\x10\x10\x11J\x04\b,\x10-J\x04\b-\x10.J\x04\b/\x100J\x04\b0\x101J\x04\b1\x102J\x04\b2\x103\"\xb5\x01\n" +
|
||||
"\x10TimeSkippingInfo\x12D\n" +
|
||||
"\x06config\x18\x01 \x01(\v2,.temporal.api.workflow.v1.TimeSkippingConfigR\x06config\x12[\n" +
|
||||
"\x1caccumulated_skipped_duration\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\x1aaccumulatedSkippedDuration\"\x7f\n" +
|
||||
"\x1caccumulated_skipped_duration\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\x1aaccumulatedSkippedDuration\"\xa8\x01\n" +
|
||||
"\x19LastNotifiedTargetVersion\x12b\n" +
|
||||
"\x12deployment_version\x18\x01 \x01(\v23.temporal.api.deployment.v1.WorkerDeploymentVersionR\x11deploymentVersion\"\x9d\x01\n" +
|
||||
"\x12deployment_version\x18\x01 \x01(\v23.temporal.api.deployment.v1.WorkerDeploymentVersionR\x11deploymentVersion\x12'\n" +
|
||||
"\x0frevision_number\x18\x02 \x01(\x03R\x0erevisionNumber\"\x9d\x01\n" +
|
||||
"\x0eExecutionStats\x12!\n" +
|
||||
"\fhistory_size\x18\x01 \x01(\x03R\vhistorySize\x122\n" +
|
||||
"\x15external_payload_size\x18\x02 \x01(\x03R\x13externalPayloadSize\x124\n" +
|
||||
|
||||
2
go.mod
2
go.mod
@@ -63,7 +63,7 @@ require (
|
||||
go.opentelemetry.io/otel/sdk v1.43.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0
|
||||
go.opentelemetry.io/otel/trace v1.43.0
|
||||
go.temporal.io/api v1.62.10-0.20260420202918-975b82732988
|
||||
go.temporal.io/api v1.62.10-0.20260421204157-0617d4e3bba2
|
||||
go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2
|
||||
go.temporal.io/sdk v1.41.1
|
||||
go.uber.org/fx v1.24.0
|
||||
|
||||
4
go.sum
4
go.sum
@@ -469,8 +469,8 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R
|
||||
go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U=
|
||||
go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0=
|
||||
go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4=
|
||||
go.temporal.io/api v1.62.10-0.20260420202918-975b82732988 h1:ZJ1SLhzqMz62LR0nIcle5MSHJzqERBGj/tsxQfX8YTo=
|
||||
go.temporal.io/api v1.62.10-0.20260420202918-975b82732988/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM=
|
||||
go.temporal.io/api v1.62.10-0.20260421204157-0617d4e3bba2 h1:9s2PjMyiiRg49fmjgWDo5RIx2MuWu5K4S8a7pThNpzU=
|
||||
go.temporal.io/api v1.62.10-0.20260421204157-0617d4e3bba2/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM=
|
||||
go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2 h1:1hKeH3GyR6YD6LKMHGCZ76t6h1Sgha0hXVQBxWi3dlQ=
|
||||
go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2/go.mod h1:T8dnzVPeO+gaUTj9eDgm/lT2lZH4+JXNvrGaQGyVi50=
|
||||
go.temporal.io/sdk v1.41.1 h1:yOpvsHyDD1lNuwlGBv/SUodCPhjv9nDeC9lLHW/fJUA=
|
||||
|
||||
@@ -333,6 +333,10 @@ message TimeSkippingInfo {
|
||||
// Used only within server persistence; never flows to the public API.
|
||||
message LastNotifiedTargetVersion {
|
||||
temporal.api.deployment.v1.WorkerDeploymentVersion deployment_version = 1;
|
||||
// Revision number of the task queue routing config at the time the
|
||||
// notification was sent. Carried forward to DeclinedTargetVersionUpgrade
|
||||
// at continue-as-new time.
|
||||
int64 revision_number = 2;
|
||||
}
|
||||
|
||||
message ExecutionStats {
|
||||
|
||||
@@ -113,6 +113,7 @@ func NewWorkflowWithSignal(
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
// Unable to add WorkflowTaskStarted event to history
|
||||
|
||||
@@ -170,6 +170,7 @@ func Invoke(
|
||||
workflowLease.GetContext().UpdateRegistry(ctx),
|
||||
false,
|
||||
req.TargetDeploymentVersion,
|
||||
req.TaskDispatchRevisionNumber,
|
||||
)
|
||||
if err != nil {
|
||||
// Unable to add WorkflowTaskStarted event to history
|
||||
|
||||
@@ -603,6 +603,7 @@ func (handler *WorkflowTaskCompletedHandler) Invoke(
|
||||
workflowLease.GetContext().UpdateRegistry(ctx),
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -728,6 +729,7 @@ func (handler *WorkflowTaskCompletedHandler) Invoke(
|
||||
workflowLease.GetContext().UpdateRegistry(ctx),
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -728,6 +728,7 @@ func (s *WorkflowTaskCompletedHandlerSuite) createSentUpdate(tv *testvars.TestVa
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
taskToken := &tokenspb.Task{
|
||||
Attempt: 1,
|
||||
@@ -807,6 +808,7 @@ func (s *WorkflowTaskCompletedHandlerSuite) createPausedWorkflowWithWFT(tv *test
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
_, _ = ms.AddWorkflowTaskCompletedEvent(wt, &workflowservice.RespondWorkflowTaskCompletedRequest{
|
||||
Identity: tv.Any().String(),
|
||||
@@ -824,6 +826,7 @@ func (s *WorkflowTaskCompletedHandlerSuite) createPausedWorkflowWithWFT(tv *test
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
taskToken := &tokenspb.Task{
|
||||
Attempt: 1,
|
||||
|
||||
@@ -238,6 +238,7 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
wt.StartedEventID = workflowTasksStartEvent.GetEventId()
|
||||
|
||||
|
||||
@@ -6618,6 +6618,7 @@ func addWorkflowTaskStartedEventWithRequestID(ms historyi.MutableState, schedule
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
|
||||
return event
|
||||
|
||||
@@ -76,7 +76,7 @@ type (
|
||||
AddFirstWorkflowTaskScheduled(parentClock *clockspb.VectorClock, event *historypb.HistoryEvent, bypassTaskGeneration bool) (int64, error)
|
||||
AddWorkflowTaskScheduledEvent(bypassTaskGeneration bool, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error)
|
||||
AddWorkflowTaskScheduledEventAsHeartbeat(bypassTaskGeneration bool, originalScheduledTimestamp *timestamppb.Timestamp, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error)
|
||||
AddWorkflowTaskStartedEvent(int64, string, *taskqueuepb.TaskQueue, string, *commonpb.WorkerVersionStamp, *taskqueuespb.BuildIdRedirectInfo, update.Registry, bool, *deploymentpb.WorkerDeploymentVersion) (*historypb.HistoryEvent, *WorkflowTaskInfo, error)
|
||||
AddWorkflowTaskStartedEvent(int64, string, *taskqueuepb.TaskQueue, string, *commonpb.WorkerVersionStamp, *taskqueuespb.BuildIdRedirectInfo, update.Registry, bool, *deploymentpb.WorkerDeploymentVersion, int64) (*historypb.HistoryEvent, *WorkflowTaskInfo, error)
|
||||
AddWorkflowTaskTimedOutEvent(workflowTask *WorkflowTaskInfo) (*historypb.HistoryEvent, error)
|
||||
AddExternalWorkflowExecutionCancelRequested(int64, namespace.Name, namespace.ID, string, string) (*historypb.HistoryEvent, error)
|
||||
AddExternalWorkflowExecutionSignaled(int64, namespace.Name, namespace.ID, string, string, string) (*historypb.HistoryEvent, error)
|
||||
|
||||
@@ -948,9 +948,9 @@ func (mr *MockMutableStateMockRecorder) AddWorkflowTaskScheduledEventAsHeartbeat
|
||||
}
|
||||
|
||||
// AddWorkflowTaskStartedEvent mocks base method.
|
||||
func (m *MockMutableState) AddWorkflowTaskStartedEvent(arg0 int64, arg1 string, arg2 *taskqueue.TaskQueue, arg3 string, arg4 *common.WorkerVersionStamp, arg5 *taskqueue0.BuildIdRedirectInfo, arg6 update0.Registry, arg7 bool, arg8 *deployment.WorkerDeploymentVersion) (*history.HistoryEvent, *WorkflowTaskInfo, error) {
|
||||
func (m *MockMutableState) AddWorkflowTaskStartedEvent(arg0 int64, arg1 string, arg2 *taskqueue.TaskQueue, arg3 string, arg4 *common.WorkerVersionStamp, arg5 *taskqueue0.BuildIdRedirectInfo, arg6 update0.Registry, arg7 bool, arg8 *deployment.WorkerDeploymentVersion, arg9 int64) (*history.HistoryEvent, *WorkflowTaskInfo, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "AddWorkflowTaskStartedEvent", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8)
|
||||
ret := m.ctrl.Call(m, "AddWorkflowTaskStartedEvent", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9)
|
||||
ret0, _ := ret[0].(*history.HistoryEvent)
|
||||
ret1, _ := ret[1].(*WorkflowTaskInfo)
|
||||
ret2, _ := ret[2].(error)
|
||||
@@ -958,9 +958,9 @@ func (m *MockMutableState) AddWorkflowTaskStartedEvent(arg0 int64, arg1 string,
|
||||
}
|
||||
|
||||
// AddWorkflowTaskStartedEvent indicates an expected call of AddWorkflowTaskStartedEvent.
|
||||
func (mr *MockMutableStateMockRecorder) AddWorkflowTaskStartedEvent(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 any) *gomock.Call {
|
||||
func (mr *MockMutableStateMockRecorder) AddWorkflowTaskStartedEvent(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9 any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowTaskStartedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowTaskStartedEvent), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8)
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowTaskStartedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowTaskStartedEvent), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9)
|
||||
}
|
||||
|
||||
// AddWorkflowTaskTimedOutEvent mocks base method.
|
||||
|
||||
@@ -541,6 +541,7 @@ func (r *workflowResetterImpl) failWorkflowTask(
|
||||
// skipping versioning checks because this task is not actually dispatched but will fail immediately.
|
||||
true,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -397,6 +397,7 @@ func (s *workflowResetterSuite) TestFailWorkflowTask_WorkflowTaskScheduled() {
|
||||
nil,
|
||||
true,
|
||||
nil,
|
||||
gomock.Any(),
|
||||
).Return(&historypb.HistoryEvent{}, workflowTaskStart, nil)
|
||||
mutableState.EXPECT().AddWorkflowTaskFailedEvent(
|
||||
workflowTaskStart,
|
||||
@@ -1553,6 +1554,7 @@ func (s *workflowResetterSuite) TestWorkflowRestartAfterExecutionTimeout() {
|
||||
nil,
|
||||
true,
|
||||
nil,
|
||||
gomock.Any(),
|
||||
).Return(&historypb.HistoryEvent{}, workflowTaskStart, nil)
|
||||
|
||||
resetMutableState.EXPECT().AddWorkflowTaskFailedEvent(
|
||||
|
||||
@@ -2679,6 +2679,7 @@ func computeDeclinedTargetVersionUpgrade(info *persistencespb.WorkflowExecutionI
|
||||
if lastNotified := info.GetLastNotifiedTargetVersion(); lastNotified != nil {
|
||||
return &historypb.DeclinedTargetVersionUpgrade{
|
||||
DeploymentVersion: lastNotified.GetDeploymentVersion(),
|
||||
RevisionNumber: lastNotified.GetRevisionNumber(),
|
||||
}
|
||||
}
|
||||
return info.GetDeclinedTargetVersionUpgrade()
|
||||
@@ -3300,12 +3301,13 @@ func (ms *MutableStateImpl) AddWorkflowTaskStartedEvent(
|
||||
updateReg update.Registry,
|
||||
skipVersioningCheck bool,
|
||||
targetDeploymentVersion *deploymentpb.WorkerDeploymentVersion,
|
||||
targetRevisionNumber int64,
|
||||
) (*historypb.HistoryEvent, *historyi.WorkflowTaskInfo, error) {
|
||||
opTag := tag.WorkflowActionWorkflowTaskStarted
|
||||
if err := ms.checkMutability(opTag); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return ms.workflowTaskManager.AddWorkflowTaskStartedEvent(scheduledEventID, requestID, taskQueue, identity, versioningStamp, redirectInfo, skipVersioningCheck, updateReg, targetDeploymentVersion)
|
||||
return ms.workflowTaskManager.AddWorkflowTaskStartedEvent(scheduledEventID, requestID, taskQueue, identity, versioningStamp, redirectInfo, skipVersioningCheck, updateReg, targetDeploymentVersion, targetRevisionNumber)
|
||||
}
|
||||
|
||||
func (ms *MutableStateImpl) ApplyWorkflowTaskStartedEvent(
|
||||
|
||||
@@ -332,6 +332,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_Valid() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.Equal("b2", wft.BuildId)
|
||||
@@ -357,6 +358,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_Invalid() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
|
||||
s.ErrorAs(err, &expectedErr)
|
||||
@@ -380,6 +382,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_EmptyRedirectInfo() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
|
||||
s.ErrorAs(err, &expectedErr)
|
||||
@@ -403,6 +406,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_EmptyStamp() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
|
||||
s.ErrorAs(err, &expectedErr)
|
||||
@@ -428,6 +432,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_Sticky() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.Equal("", wft.BuildId)
|
||||
@@ -455,6 +460,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_StickyInvalid() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
|
||||
s.ErrorAs(err, &expectedErr)
|
||||
@@ -479,6 +485,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_UnexpectedSticky() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
|
||||
s.ErrorAs(err, &expectedErr)
|
||||
@@ -541,6 +548,7 @@ func (s *mutableStateSuite) TestPopulateDeleteTasks_WithWorkflowTaskTimeouts() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
|
||||
@@ -595,6 +603,7 @@ func (s *mutableStateSuite) TestPopulateDeleteTasks_LongTimeout_NotIncluded() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
|
||||
@@ -664,6 +673,7 @@ func (s *mutableStateSuite) createVersionedMutableStateWithCompletedWFT(tq *task
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.Equal("b1", wft.BuildId)
|
||||
@@ -947,6 +957,7 @@ func (s *mutableStateSuite) createMutableStateWithVersioningBehavior(
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.verifyEffectiveDeployment(deployment, enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE)
|
||||
@@ -998,6 +1009,7 @@ func (s *mutableStateSuite) TestUnpinnedTransition() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.verifyEffectiveDeployment(deployment2, behavior)
|
||||
@@ -1038,6 +1050,7 @@ func (s *mutableStateSuite) TestUnpinnedTransitionFailed() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.verifyEffectiveDeployment(deployment2, behavior)
|
||||
@@ -1081,6 +1094,7 @@ func (s *mutableStateSuite) TestUnpinnedTransitionTimeout() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.verifyEffectiveDeployment(deployment2, behavior)
|
||||
@@ -1276,6 +1290,7 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.verifyEffectiveDeployment(deployment3, overrideBehavior)
|
||||
@@ -1731,6 +1746,7 @@ func (s *mutableStateSuite) TestAddWorkflowExecutionPausedEvent() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
completedEvent, err := s.mutableState.AddWorkflowTaskCompletedEvent(
|
||||
@@ -1794,6 +1810,7 @@ func (s *mutableStateSuite) TestAddWorkflowExecutionUnpausedEvent() {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
completedEvent, err := s.mutableState.AddWorkflowTaskCompletedEvent(
|
||||
@@ -2017,6 +2034,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskStart_CurrentVersionChanged
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
s.Equal(0, s.mutableState.hBuilder.NumBufferedEvents())
|
||||
@@ -6185,6 +6203,7 @@ func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerControlTa
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
s.NoError(err)
|
||||
_, err = s.mutableState.AddWorkflowTaskCompletedEvent(
|
||||
|
||||
@@ -463,6 +463,7 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskStartedEvent(
|
||||
skipVersioningCheck bool,
|
||||
updateReg update.Registry,
|
||||
targetDeploymentVersion *deploymentpb.WorkerDeploymentVersion,
|
||||
targetRevisionNumber int64,
|
||||
) (*historypb.HistoryEvent, *historyi.WorkflowTaskInfo, error) {
|
||||
opTag := tag.WorkflowActionWorkflowTaskStarted
|
||||
workflowTask := m.GetWorkflowTaskByID(scheduledEventID)
|
||||
@@ -515,15 +516,15 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskStartedEvent(
|
||||
// TODO (Shivam): Revision number mechanics to strengthen this check
|
||||
m.ms.executionInfo.DeclinedTargetVersionUpgrade = nil
|
||||
m.ms.executionInfo.LastNotifiedTargetVersion = nil
|
||||
// 4. Previously declined upgrade — target unchanged since the decline.
|
||||
// 4. Previously declined upgrade — target revision is not newer than what was declined.
|
||||
case m.ms.executionInfo.GetDeclinedTargetVersionUpgrade() != nil &&
|
||||
m.ms.executionInfo.GetDeclinedTargetVersionUpgrade().GetDeploymentVersion().GetBuildId() == targetDeploymentVersion.GetBuildId() &&
|
||||
m.ms.executionInfo.GetDeclinedTargetVersionUpgrade().GetDeploymentVersion().GetDeploymentName() == targetDeploymentVersion.GetDeploymentName():
|
||||
targetRevisionNumber <= m.ms.executionInfo.GetDeclinedTargetVersionUpgrade().GetRevisionNumber():
|
||||
default:
|
||||
// Otherwise — target changed + did not decline to upgrade on CaN/retry. Signal the SDK.
|
||||
targetDeploymentVersionChanged = true
|
||||
m.ms.executionInfo.LastNotifiedTargetVersion = &persistencespb.LastNotifiedTargetVersion{
|
||||
DeploymentVersion: targetDeploymentVersion,
|
||||
RevisionNumber: targetRevisionNumber,
|
||||
}
|
||||
m.ms.executionInfo.DeclinedTargetVersionUpgrade = nil
|
||||
}
|
||||
|
||||
@@ -138,6 +138,7 @@ func (c *mutationTestCase) startWFT(
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -446,6 +447,7 @@ func TestGetNexusCompletion(t *testing.T) {
|
||||
nil,
|
||||
false,
|
||||
nil,
|
||||
0,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
_, err = ms.AddWorkflowTaskCompletedEvent(workflowTask, &workflowservice.RespondWorkflowTaskCompletedRequest{
|
||||
|
||||
@@ -1876,7 +1876,7 @@ func (pm *taskQueuePartitionManagerImpl) getPhysicalQueuesForAdd(
|
||||
if wfBehavior == enumspb.VERSIONING_BEHAVIOR_PINNED {
|
||||
if pm.partition.Kind() == enumspb.TASK_QUEUE_KIND_STICKY {
|
||||
// TODO (shahab): we can verify the passed deployment matches the last poller's deployment
|
||||
return dbq, dbq, userDataChanged, 0, targetDeploymentVersion, nil
|
||||
return dbq, dbq, userDataChanged, targetDeploymentRevisionNumber, targetDeploymentVersion, nil
|
||||
}
|
||||
|
||||
err = worker_versioning.ValidateDeployment(deployment)
|
||||
@@ -1906,15 +1906,15 @@ func (pm *taskQueuePartitionManagerImpl) getPhysicalQueuesForAdd(
|
||||
if !isIndependentPinnedActivity {
|
||||
pinnedQueue, err := pm.getVersionedQueue(ctx, "", "", deployment, true)
|
||||
if err != nil {
|
||||
return nil, nil, nil, 0, nil, err // TODO (Shivam): Please add the comment in the proto to explain that pinned tasks and sticky tasks get 0 for the rev number.
|
||||
return nil, nil, nil, 0, nil, err
|
||||
}
|
||||
if forwardInfo == nil {
|
||||
// Task is not forwarded, so it can be spooled if sync match fails.
|
||||
// Spool queue and sync match queue is the same for pinned workflows.
|
||||
return pinnedQueue, pinnedQueue, userDataChanged, 0, targetDeploymentVersion, nil
|
||||
return pinnedQueue, pinnedQueue, userDataChanged, targetDeploymentRevisionNumber, targetDeploymentVersion, nil
|
||||
} else {
|
||||
// Forwarded from child partition - only do sync match.
|
||||
return nil, pinnedQueue, userDataChanged, 0, targetDeploymentVersion, nil
|
||||
return nil, pinnedQueue, userDataChanged, targetDeploymentRevisionNumber, targetDeploymentVersion, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6367,6 +6367,150 @@ func (s *Versioning3Suite) TestRemoveOverride_ClearsDeclinedState() {
|
||||
})
|
||||
}
|
||||
|
||||
// TestStalePartition_RevisionSuppressesTrampolining verifies that when a stale
|
||||
// matching partition sends an outdated target version, the revision-based
|
||||
// comparison in case 4 suppresses the signal and prevents trampolining.
|
||||
//
|
||||
// Flow:
|
||||
// 1. Start pinned workflow on v1, set v1 as current
|
||||
// 2. Set v2 as current (revision increments)
|
||||
// 3. Set v3 as current (revision increments again)
|
||||
// 4. Trigger WFT → signal fires (target=v3 at high revision), CaN with decline
|
||||
// 5. Roll back task queue to v2 with revision 0 (simulating stale partition)
|
||||
// 6. Trigger WFT → assert targetDeploymentVersionChanged=false (revision 0 <= declined revision)
|
||||
// 7. Set v4 as current (fresh version with higher revision)
|
||||
// 8. Trigger WFT → assert targetDeploymentVersionChanged=true (new revision > declined revision)
|
||||
func (s *Versioning3Suite) TestStalePartition_RevisionSuppressesTrampolining() {
|
||||
s.OverrideDynamicConfig(dynamicconfig.MatchingNumTaskqueueReadPartitions, 1)
|
||||
s.OverrideDynamicConfig(dynamicconfig.MatchingNumTaskqueueWritePartitions, 1)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tv1 := testvars.New(s).WithBuildIDNumber(1)
|
||||
tv2 := tv1.WithBuildIDNumber(2)
|
||||
tv3 := tv1.WithBuildIDNumber(3)
|
||||
|
||||
// Start async poller for v1 that handles first WFT and declares pinned behavior
|
||||
wftCompleted := make(chan struct{})
|
||||
s.pollWftAndHandle(tv1, false, wftCompleted,
|
||||
func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) {
|
||||
s.NotNil(task)
|
||||
return respondEmptyWft(tv1, false, vbPinned), nil
|
||||
})
|
||||
|
||||
// Register v1 and set it as current
|
||||
s.waitForDeploymentDataPropagation(tv1, versionStatusInactive, false, tqTypeWf)
|
||||
s.setCurrentDeployment(tv1)
|
||||
|
||||
// Start workflow — first WFT handled by async poller (pinned on v1)
|
||||
runID := s.startWorkflow(tv1, nil)
|
||||
execution := tv1.WithRunID(runID).WorkflowExecution()
|
||||
s.WaitForChannel(ctx, wftCompleted)
|
||||
s.verifyWorkflowVersioning(s.Assertions, tv1, vbPinned, tv1.Deployment(), nil, nil)
|
||||
|
||||
// Register v2, set v2 as current (revision increments)
|
||||
s.idlePollWorkflow(ctx, tv2, true, ver3MinPollTime, "v2 poller registration")
|
||||
s.setCurrentDeployment(tv2)
|
||||
|
||||
// Register v3, set v3 as current (revision increments again)
|
||||
s.idlePollWorkflow(ctx, tv3, true, ver3MinPollTime, "v3 poller registration")
|
||||
s.setCurrentDeployment(tv3)
|
||||
|
||||
// Trigger WFT — target should be v3 with a high revision
|
||||
s.triggerNormalWFT(ctx, tv1, execution)
|
||||
|
||||
// Process: targetDeploymentVersionChanged=true → CaN without AU (decline v3)
|
||||
s.pollWftAndHandle(tv1, false, nil,
|
||||
func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) {
|
||||
s.NotNil(task)
|
||||
var lastStarted *historypb.HistoryEvent
|
||||
for _, event := range task.History.GetEvents() {
|
||||
if event.GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED {
|
||||
lastStarted = event
|
||||
}
|
||||
}
|
||||
s.NotNil(lastStarted)
|
||||
s.True(lastStarted.GetWorkflowTaskStartedEventAttributes().GetTargetWorkerDeploymentVersionChanged(),
|
||||
"expected true after v3 becomes current")
|
||||
|
||||
return &workflowservice.RespondWorkflowTaskCompletedRequest{
|
||||
Commands: []*commandpb.Command{
|
||||
{
|
||||
CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION,
|
||||
Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{
|
||||
ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{
|
||||
WorkflowType: tv1.WorkflowType(),
|
||||
TaskQueue: tv1.TaskQueue(),
|
||||
Input: tv1.Any().Payloads(),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
VersioningBehavior: vbPinned,
|
||||
DeploymentOptions: tv1.WorkerDeploymentOptions(true),
|
||||
}, nil
|
||||
})
|
||||
|
||||
// CaN run: first WFT — declined=v3 propagated from previous run
|
||||
s.pollWftAndHandle(tv1, false, nil,
|
||||
func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) {
|
||||
s.NotNil(task)
|
||||
s.NotEqual(execution.RunId, task.WorkflowExecution.RunId,
|
||||
"CaN should have created a new run")
|
||||
execution = task.WorkflowExecution
|
||||
return respondEmptyWft(tv1, false, vbPinned), nil
|
||||
})
|
||||
|
||||
// Simulate stale partition: roll back task queue to v2 with revision 0
|
||||
s.rollbackTaskQueueToVersion(tv2)
|
||||
|
||||
// Trigger WFT with stale data — target is now v2 at revision 0
|
||||
s.triggerNormalWFT(ctx, tv1, execution)
|
||||
|
||||
// Assert: revision 0 <= declined revision → signal suppressed
|
||||
s.pollWftAndHandle(tv1, false, nil,
|
||||
func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) {
|
||||
s.NotNil(task)
|
||||
var lastStarted *historypb.HistoryEvent
|
||||
for _, event := range task.History.GetEvents() {
|
||||
if event.GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED {
|
||||
lastStarted = event
|
||||
}
|
||||
}
|
||||
s.NotNil(lastStarted)
|
||||
s.False(lastStarted.GetWorkflowTaskStartedEventAttributes().GetTargetWorkerDeploymentVersionChanged(),
|
||||
"stale partition (revision 0) should be suppressed by declined revision")
|
||||
return respondEmptyWft(tv1, false, vbPinned), nil
|
||||
})
|
||||
|
||||
// Set a new v4 as current — this produces a revision strictly higher than
|
||||
// the declined revision, simulating an up-to-date partition with fresh data.
|
||||
tv4 := tv1.WithBuildIDNumber(4)
|
||||
s.idlePollWorkflow(ctx, tv4, true, ver3MinPollTime, "v4 poller registration")
|
||||
s.setCurrentDeployment(tv4)
|
||||
s.waitForDeploymentDataPropagation(tv4, versionStatusCurrent, false, tqTypeWf)
|
||||
|
||||
// Trigger WFT with fresh data — target is v4 at higher revision
|
||||
s.triggerNormalWFT(ctx, tv1, execution)
|
||||
|
||||
// Assert: new revision > declined revision → signal fires
|
||||
s.pollWftAndHandle(tv1, false, nil,
|
||||
func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) {
|
||||
s.NotNil(task)
|
||||
var lastStarted *historypb.HistoryEvent
|
||||
for _, event := range task.History.GetEvents() {
|
||||
if event.GetEventType() == enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED {
|
||||
lastStarted = event
|
||||
}
|
||||
}
|
||||
s.NotNil(lastStarted)
|
||||
s.True(lastStarted.GetWorkflowTaskStartedEventAttributes().GetTargetWorkerDeploymentVersionChanged(),
|
||||
"up-to-date partition with higher revision should fire signal")
|
||||
return respondCompleteWorkflow(tv1, vbPinned), nil
|
||||
})
|
||||
}
|
||||
|
||||
// TestRetryOfDeclinedCaN_SignalsOnNewTarget verifies that when a CaN'd run
|
||||
// ,which declined to upgrade, fails and is retried by the server, the retry
|
||||
// run inherits NotificationSuppressedTargetVersion from the original CaN
|
||||
|
||||
Reference in New Issue
Block a user