Files
temporal/common/persistence/serialization/task_serializers_test.go
jiechenz 72567b9ed1 Replicate workflow deleteion (#9717)
## What changed?

Add a new replication task type `DeleteExecutionReplicationTask` that
replicates workflow deletion from the active cluster to passive/standby
clusters. Gated by feature flag
`history.enableDeleteWorkflowExecutionReplication` (default: false).

#### **Key changes across the replication pipeline:**
1. Proto enums: `TASK_TYPE_REPLICATION_DELETE_EXECUTION` (34),
`REPLICATION_TASK_TYPE_DELETE_EXECUTION_TASK` (13)
2. Replication task is associated with a new stage in
`ShardContext.DeleteWorkflowExecution`, bundled with delete visibility
task.
3. ~Engine interface: added `ForceDeleteWorkflowExecution` so the task
can invoke the `ForceDeleteWorkflowExecution`.~

## Why?

Today, when a user delete workflow execution in source cluster, this
operation will not replicate to the standby/target clusters. When a
namespace failover to a target cluster, those deleted workflow may
resurrected.

  <details>
  <summary>Race condition analysis</summary>

  **Before this change:**

1. **Cross-cluster resurrection:** Active deletes workflow → standby
untouched → failover → workflow reappears.
2. **Termination event silently dropped:** Deleting a running workflow
terminates it first, generating a `HistoryReplicationTask`. But the
async `CloseExecutionTask` may delete mutable state before the stream
sender converts that task.
The converter calls `getBranchToken()` → `NotFound` → task silently
dropped. The standby never sees the termination or the deletion.

  **After this change:**

Race 1 is fixed — `DeleteExecutionReplicationTask` explicitly tells the
standby to delete.

Race 2 is mitigated — even if the termination event's replication task
is dropped, the delete replication task ensures the standby cleans up.
- If the workflow is still running (termination not yet replicated), the
`DeleteExecutionTask` reschedules itself until the workflow closes.
- If the termination event arrives later, the workflow closes normally,
then the delete proceeds.
- If the workflow is already deleted (e.g., by retention), the task is a
no-op (`NotFound` treated as success).

  </details>

  <details>
  <summary>Deletion paths</summary>

  | Path | Replication task? |
  |------|-------------------|
  | User deletes workflow (active, running or closed) | Yes |
| User deletes on passive (DC forwarding ON) | Forwarded to active → yes
|
| User deletes on passive (no forwarding) | No — `ActiveInCluster` check
skips |
| Retention expiry (with or without archival) | No — stage pre-marked as
processed |
  | Admin ForceDelete (tdbg) | No — bypasses `DeleteWorkflowExecution` |

  </details>

## How did you test it?
- [x] built
- [x] run locally and tested manually
- [ ] covered by existing tests
- [x] added new unit test(s)
- [x] added new functional test(s)

Before change:
<img width="1507" height="163" alt="Screenshot 2026-03-26 at 11 59
51 PM"
src="https://github.com/user-attachments/assets/118cc50e-b69d-468a-9e45-5f49e4e4b9d1"
/>

After change:
<img width="1507" height="135" alt="Screenshot 2026-03-27 at 12 00
07 AM"
src="https://github.com/user-attachments/assets/8ccb7a11-2cb4-48b5-af89-b4a60ddb6333"
/>



## Potential risks
n/a
2026-04-02 14:49:09 -07:00

611 lines
19 KiB
Go

package serialization
import (
"math/rand"
"reflect"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
commonpb "go.temporal.io/api/common/v1"
enumspb "go.temporal.io/api/enums/v1"
enumsspb "go.temporal.io/server/api/enums/v1"
persistencespb "go.temporal.io/server/api/persistence/v1"
"go.temporal.io/server/common/definition"
"go.temporal.io/server/common/shuffle"
"go.temporal.io/server/common/testing/protorequire"
"go.temporal.io/server/service/history/tasks"
"google.golang.org/protobuf/proto"
)
type (
taskSerializerSuite struct {
suite.Suite
*require.Assertions
workflowKey definition.WorkflowKey
serializer Serializer
}
)
func TestTaskSerializerSuite(t *testing.T) {
suite.Run(t, new(taskSerializerSuite))
}
func (s *taskSerializerSuite) SetupSuite() {
}
func (s *taskSerializerSuite) TearDownSuite() {
}
func (s *taskSerializerSuite) SetupTest() {
s.Assertions = require.New(s.T())
s.workflowKey = definition.NewWorkflowKey(
"random namespace ID",
"random workflow ID",
"random run ID",
)
s.serializer = NewSerializer()
}
func (s *taskSerializerSuite) TearDownTest() {
}
func (s *taskSerializerSuite) TestTransferWorkflowTask() {
workflowTask := &tasks.WorkflowTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
TaskQueue: shuffle.String("random task queue name"),
ScheduledEventID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(workflowTask)
}
func (s *taskSerializerSuite) TestTransferActivityTask() {
activityTask := &tasks.ActivityTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
TaskQueue: shuffle.String("random task queue name"),
ScheduledEventID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(activityTask)
}
func (s *taskSerializerSuite) TestTransferRequestCancelTask() {
requestCancelTask := &tasks.CancelExecutionTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
TargetNamespaceID: uuid.New().String(),
TargetWorkflowID: uuid.New().String(),
TargetRunID: uuid.New().String(),
TargetChildWorkflowOnly: rand.Int63()%2 == 0,
InitiatedEventID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(requestCancelTask)
}
func (s *taskSerializerSuite) TestTransferSignalTask() {
signalTask := &tasks.SignalExecutionTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
TargetNamespaceID: uuid.New().String(),
TargetWorkflowID: uuid.New().String(),
TargetRunID: uuid.New().String(),
TargetChildWorkflowOnly: rand.Int63()%2 == 0,
InitiatedEventID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(signalTask)
}
func (s *taskSerializerSuite) TestTransferChildWorkflowTask() {
childWorkflowTask := &tasks.StartChildExecutionTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
TargetNamespaceID: uuid.New().String(),
TargetWorkflowID: uuid.New().String(),
InitiatedEventID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(childWorkflowTask)
}
func (s *taskSerializerSuite) TestTransferChasmTask() {
transferChasmTask := &tasks.ChasmTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Category: tasks.CategoryTransfer,
Info: &persistencespb.ChasmTaskInfo{
Data: &commonpb.DataBlob{
Data: []byte("some-data"),
},
ArchetypeId: rand.Uint32(),
},
}
s.assertEqualTasks(transferChasmTask)
}
func (s *taskSerializerSuite) TestTransferCloseTask() {
closeTask := &tasks.CloseExecutionTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(closeTask)
}
func (s *taskSerializerSuite) TestTransferResetTask() {
resetTask := &tasks.ResetWorkflowTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(resetTask)
}
func (s *taskSerializerSuite) TestTimerWorkflowTask() {
workflowTaskTimer := &tasks.WorkflowTaskTimeoutTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
EventID: rand.Int63(),
ScheduleAttempt: rand.Int31(),
TimeoutType: enumspb.TimeoutType(rand.Int31n(int32(len(enumspb.TimeoutType_name)))),
Version: rand.Int63(),
}
s.assertEqualTasks(workflowTaskTimer)
}
func (s *taskSerializerSuite) TestTimerWorkflowDelayTask() {
workflowDelayTimer := &tasks.WorkflowBackoffTimerTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
WorkflowBackoffType: enumsspb.WorkflowBackoffType(rand.Int31n(int32(len(enumsspb.WorkflowBackoffType_name)))),
Version: rand.Int63(),
}
s.assertEqualTasks(workflowDelayTimer)
}
func (s *taskSerializerSuite) TestTimerActivityTask() {
activityTaskTimer := &tasks.ActivityTimeoutTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
EventID: rand.Int63(),
Attempt: rand.Int31(),
TimeoutType: enumspb.TimeoutType(rand.Int31n(int32(len(enumspb.TimeoutType_name)))),
}
s.assertEqualTasks(activityTaskTimer)
}
func (s *taskSerializerSuite) TestTimerActivityRetryTask() {
activityRetryTimer := &tasks.ActivityRetryTimerTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
EventID: rand.Int63(),
Attempt: rand.Int31(),
Version: rand.Int63(),
}
s.assertEqualTasks(activityRetryTimer)
}
func (s *taskSerializerSuite) TestTimerUserTask() {
userTimer := &tasks.UserTimerTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
EventID: rand.Int63(),
}
s.assertEqualTasks(userTimer)
}
func (s *taskSerializerSuite) TestTimerWorkflowRun() {
workflowRunTimer := &tasks.WorkflowRunTimeoutTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(workflowRunTimer)
}
func (s *taskSerializerSuite) TestTimerWorkflowExecution() {
workflowExecutionTimer := &tasks.WorkflowExecutionTimeoutTask{
NamespaceID: s.workflowKey.NamespaceID,
WorkflowID: s.workflowKey.WorkflowID,
FirstRunID: s.workflowKey.RunID,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
}
s.assertEqualTasks(workflowExecutionTimer)
}
func (s *taskSerializerSuite) TestTimerWorkflowCleanupTask() {
workflowCleanupTimer := &tasks.DeleteHistoryEventTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Version: rand.Int63(),
BranchToken: []byte{123},
ArchetypeID: rand.Uint32(),
}
s.assertEqualTasks(workflowCleanupTimer)
}
func (s *taskSerializerSuite) TestVisibilityStartTask() {
visibilityStart := &tasks.StartExecutionVisibilityTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(visibilityStart)
}
func (s *taskSerializerSuite) TestVisibilityUpsertTask() {
visibilityUpsert := &tasks.UpsertExecutionVisibilityTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
}
s.assertEqualTasks(visibilityUpsert)
}
func (s *taskSerializerSuite) TestVisibilityCloseTask() {
visibilityClose := &tasks.CloseExecutionVisibilityTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Version: rand.Int63(),
}
s.assertEqualTasks(visibilityClose)
}
func (s *taskSerializerSuite) TestDeleteExecutionVisibilityTask() {
deleteExecutionVisibilityTask := &tasks.DeleteExecutionVisibilityTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
ArchetypeID: rand.Uint32(),
CloseExecutionVisibilityTaskID: rand.Int63(),
CloseTime: time.Unix(0, 0).UTC(),
}
s.assertEqualTasks(deleteExecutionVisibilityTask)
}
func (s *taskSerializerSuite) TestVisibilityChasmTask() {
visibilityChasmTask := &tasks.ChasmTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Category: tasks.CategoryVisibility,
Info: &persistencespb.ChasmTaskInfo{
Data: &commonpb.DataBlob{
Data: []byte("some-data"),
},
ArchetypeId: rand.Uint32(),
},
}
s.assertEqualTasks(visibilityChasmTask)
}
func (s *taskSerializerSuite) TestSyncActivityTask() {
syncActivityTask := &tasks.SyncActivityTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
Version: rand.Int63(),
ScheduledEventID: rand.Int63(),
}
s.assertEqualTasks(syncActivityTask)
}
func (s *taskSerializerSuite) TestHistoryReplicationTask() {
historyReplicationTask := &tasks.HistoryReplicationTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
Version: rand.Int63(),
FirstEventID: rand.Int63(),
NextEventID: rand.Int63(),
BranchToken: shuffle.Bytes([]byte("random branch token")),
NewRunBranchToken: shuffle.Bytes([]byte("random new branch token")),
}
s.assertEqualTasks(historyReplicationTask)
}
func (s *taskSerializerSuite) TestSyncHSMTask() {
syncHSMTask := &tasks.SyncHSMTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
}
s.assertEqualTasks(syncHSMTask)
}
func (s *taskSerializerSuite) TestSyncVersionedTransitionTask() {
syncVersionedTransitionTask := &tasks.SyncVersionedTransitionTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
ArchetypeID: rand.Uint32(),
FirstEventID: rand.Int63(),
NextEventID: rand.Int63(),
NewRunID: uuid.New().String(),
VersionedTransition: &persistencespb.VersionedTransition{
NamespaceFailoverVersion: rand.Int63(),
TransitionCount: rand.Int63(),
},
TaskEquivalents: []tasks.Task{
&tasks.HistoryReplicationTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(),
FirstEventID: rand.Int63(),
NextEventID: rand.Int63(),
Version: rand.Int63(),
NewRunID: uuid.New().String(),
},
},
}
s.assertEqualTasksWithOpts(syncVersionedTransitionTask,
func(task, deserializedTask tasks.Task) {
s.True(proto.Equal(task.(*tasks.SyncVersionedTransitionTask).VersionedTransition, deserializedTask.(*tasks.SyncVersionedTransitionTask).VersionedTransition))
},
cmpopts.IgnoreFields(tasks.SyncVersionedTransitionTask{}, "VersionedTransition"),
)
}
func (s *taskSerializerSuite) TestSyncWorkflowStateTask() {
syncWorkflowStateTask := &tasks.SyncWorkflowStateTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
Version: rand.Int63(),
Priority: enumsspb.TASK_PRIORITY_LOW,
}
s.assertEqualTasks(syncWorkflowStateTask)
}
func (s *taskSerializerSuite) TestDeleteExecutionReplicationTask() {
deleteExecutionReplicationTask := &tasks.DeleteExecutionReplicationTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
ArchetypeID: rand.Uint32(),
}
s.assertEqualTasks(deleteExecutionReplicationTask)
}
func (s *taskSerializerSuite) TestDeleteExecutionTask() {
deleteExecutionTask := &tasks.DeleteExecutionTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
ArchetypeID: rand.Uint32(),
}
s.assertEqualTasks(deleteExecutionTask)
}
func (s *taskSerializerSuite) TestArchiveExecutionTask() {
task := &tasks.ArchiveExecutionTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling
TaskID: rand.Int63(),
Version: rand.Int63(),
}
s.Assert().Equal(tasks.CategoryArchival, task.GetCategory())
s.Assert().Equal(enumsspb.TASK_TYPE_ARCHIVAL_ARCHIVE_EXECUTION, task.GetType())
s.assertEqualTasks(task)
}
func (s *taskSerializerSuite) TestOutboundChasmTask() {
task := &tasks.ChasmTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Category: tasks.CategoryOutbound,
Info: &persistencespb.ChasmTaskInfo{
Data: &commonpb.DataBlob{
Data: []byte("some-data"),
},
ArchetypeId: rand.Uint32(),
},
Destination: "somewhere",
}
s.assertEqualTasks(task)
}
func (s *taskSerializerSuite) TestStateMachineOutboundTask() {
task := &tasks.StateMachineOutboundTask{
StateMachineTask: tasks.StateMachineTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Now().UTC(),
TaskID: rand.Int63(),
Info: &persistencespb.StateMachineTaskInfo{
Ref: &persistencespb.StateMachineRef{
Path: []*persistencespb.StateMachineKey{
{
Type: "some-type",
Id: "some-id",
},
},
MutableStateVersionedTransition: &persistencespb.VersionedTransition{
NamespaceFailoverVersion: rand.Int63(),
TransitionCount: rand.Int63(),
},
MachineInitialVersionedTransition: &persistencespb.VersionedTransition{
NamespaceFailoverVersion: rand.Int63(),
TransitionCount: rand.Int63(),
},
MachineLastUpdateVersionedTransition: &persistencespb.VersionedTransition{
NamespaceFailoverVersion: rand.Int63(),
TransitionCount: rand.Int63(),
},
MachineTransitionCount: rand.Int63(),
},
Type: "some-type",
Data: []byte{},
},
},
Destination: "foo",
}
s.Assert().Equal(tasks.CategoryOutbound, task.GetCategory())
s.Assert().Equal(enumsspb.TASK_TYPE_STATE_MACHINE_OUTBOUND, task.GetType())
blob, err := s.serializer.SerializeTask(task)
s.NoError(err)
deserializedTaskIface, err := s.serializer.DeserializeTask(task.GetCategory(), blob)
deserializedTask := deserializedTaskIface.(*tasks.StateMachineOutboundTask)
s.NoError(err)
protorequire.ProtoEqual(s.T(), task.Info, deserializedTask.Info)
task.Info = nil
deserializedTask.Info = nil
s.Equal(task, deserializedTask)
}
func (s *taskSerializerSuite) TestTimerChasmTask() {
task := &tasks.ChasmTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
Category: tasks.CategoryTimer,
Info: &persistencespb.ChasmTaskInfo{
Data: &commonpb.DataBlob{
Data: []byte("some-data"),
},
ArchetypeId: rand.Uint32(),
},
}
s.assertEqualTasks(task)
}
func (s *taskSerializerSuite) TestTimerChasmPureTask() {
task := &tasks.ChasmTaskPure{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Unix(0, rand.Int63()).UTC(),
TaskID: rand.Int63(),
ArchetypeID: rand.Uint32(),
}
s.assertEqualTasks(task)
}
func (s *taskSerializerSuite) TestStateMachineTimerTask() {
task := &tasks.StateMachineTimerTask{
WorkflowKey: s.workflowKey,
VisibilityTimestamp: time.Now().UTC(),
TaskID: rand.Int63(),
Version: rand.Int63(),
}
s.Assert().Equal(tasks.CategoryTimer, task.GetCategory())
s.Assert().Equal(enumsspb.TASK_TYPE_STATE_MACHINE_TIMER, task.GetType())
blob, err := s.serializer.SerializeTask(task)
s.NoError(err)
deserializedTaskIface, err := s.serializer.DeserializeTask(task.GetCategory(), blob)
deserializedTask := deserializedTaskIface.(*tasks.StateMachineTimerTask)
s.NoError(err)
s.Equal(task, deserializedTask)
}
func (s *taskSerializerSuite) assertEqualTasksWithOpts(
task tasks.Task,
cmpFunc func(task, deserializedTask tasks.Task),
opts ...cmp.Option,
) {
blob, err := s.serializer.SerializeTask(task)
s.NoError(err)
deserializedTask, err := s.serializer.DeserializeTask(task.GetCategory(), blob)
s.NoError(err)
s.Empty(cmp.Diff(task, deserializedTask, opts...))
if cmpFunc != nil {
cmpFunc(task, deserializedTask)
}
}
func (s *taskSerializerSuite) assertEqualTasks(
task tasks.Task,
) {
blob, err := s.serializer.SerializeTask(task)
s.NoError(err)
deserializedTask, err := s.serializer.DeserializeTask(task.GetCategory(), blob)
s.NoError(err)
// Find all top-level protobuf fields and compare them, then unset them, as their
// internal state parameters can cause direct struct comparison to fail.
taskValue := reflect.ValueOf(task).Elem()
deserializedTaskValue := reflect.ValueOf(deserializedTask).Elem()
s.Equal(taskValue.Type(), deserializedTaskValue.Type())
for i := range deserializedTaskValue.NumField() {
deField := deserializedTaskValue.Field(i)
if !deField.CanInterface() {
continue
}
if innerProto, ok := deField.Interface().(proto.Message); ok {
originalProto, _ := taskValue.Field(i).Interface().(proto.Message)
protorequire.ProtoEqual(s.T(), originalProto, innerProto)
taskValue.Field(i).SetZero()
deserializedTaskValue.Field(i).SetZero()
}
}
s.Equal(task, deserializedTask)
}