diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index db0da0a964..0a9630df8d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -534,8 +534,6 @@ jobs: run: ./develop/github/monitor_test.sh make functional-test-coverage env: TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }} - TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput - TEMPORAL_OTEL_DEBUG: true - name: Print memory snapshot if: always() @@ -595,17 +593,6 @@ jobs: FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_test_suites }} -count=5" - # Upload OpenTelemetry traces. - - name: Upload OpenTelemetry traces - if: always() - uses: actions/upload-artifact@v4 - with: - name: traces-json--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{matrix.shard_index}}--${{github.run_attempt}}--${{matrix.name}}--functional-test - path: ./.testoutput/traces.*.json - if-no-files-found: ignore - include-hidden-files: true - retention-days: 28 - # XDC matrix job. Include `${{ matrix.name }}` in the display name so each # matrix variant has a unique name for the job-id lookup. functional-test-xdc: @@ -708,9 +695,6 @@ jobs: - name: Run functional test xdc timeout-minutes: 35 # update this to TEST_TIMEOUT+5 if you update the Makefile run: ./develop/github/monitor_test.sh make functional-test-xdc-coverage - env: - TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput - TEMPORAL_OTEL_DEBUG: true - name: Print memory snapshot if: always() @@ -770,16 +754,6 @@ jobs: FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_xdc_test_suites }} -count=5" - # Upload OpenTelemetry traces. - - name: Upload OpenTelemetry traces - if: always() - uses: actions/upload-artifact@v4 - with: - name: traces-json--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{github.run_attempt}}--${{matrix.name}}--functional-test-xdc - path: ./.testoutput/traces.*.json - if-no-files-found: ignore - retention-days: 28 - # NDC matrix job. Include `${{ matrix.name }}` in the display name so each # matrix variant has a unique name for the job-id lookup. functional-test-ndc: @@ -879,9 +853,6 @@ jobs: - name: Run functional test ndc timeout-minutes: 15 run: ./develop/github/monitor_test.sh make functional-test-ndc-coverage - env: - TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput - TEMPORAL_OTEL_DEBUG: true - name: Print memory snapshot if: always() @@ -931,16 +902,6 @@ jobs: FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_ndc_test_suites }} -count=5" - # Upload OpenTelemetry traces. - - name: Upload OpenTelemetry traces - if: always() - uses: actions/upload-artifact@v4 - with: - name: traces-json--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{github.run_attempt}}--${{matrix.name}}--functional-test-ndc - path: ./.testoutput/traces.*.json - if-no-files-found: ignore - retention-days: 28 - test-status: if: always() name: Test Status diff --git a/common/persistence/persistence-tests/persistence_test_base.go b/common/persistence/persistence-tests/persistence_test_base.go index e0699e490c..e53079a3eb 100644 --- a/common/persistence/persistence-tests/persistence_test_base.go +++ b/common/persistence/persistence-tests/persistence_test_base.go @@ -1,6 +1,7 @@ package persistencetests import ( + "cmp" "context" "fmt" "math/rand" @@ -61,7 +62,24 @@ type ( FaultInjection *config.FaultInjection Logger log.Logger `yaml:"-"` } +) +// ApplyDefaults copies database configuration from src, preserving any non-zero values already set. +func (o *TestBaseOptions) ApplyDefaults(src *TestBaseOptions) { + o.StoreType = cmp.Or(o.StoreType, src.StoreType) + o.SQLDBPluginName = cmp.Or(o.SQLDBPluginName, src.SQLDBPluginName) + o.DBName = cmp.Or(o.DBName, src.DBName) + o.DBUsername = cmp.Or(o.DBUsername, src.DBUsername) + o.DBPassword = cmp.Or(o.DBPassword, src.DBPassword) + o.DBHost = cmp.Or(o.DBHost, src.DBHost) + o.DBPort = cmp.Or(o.DBPort, src.DBPort) + o.SchemaDir = cmp.Or(o.SchemaDir, src.SchemaDir) + if o.ConnectAttributes == nil { + o.ConnectAttributes = src.ConnectAttributes + } +} + +type ( // TestBase wraps the base setup needed to create workflows over persistence layer. TestBase struct { suite.Suite @@ -119,9 +137,6 @@ func NewTestClusterForCassandra(options *TestBaseOptions, logger log.Logger) *ca // NewTestBaseWithSQL returns a new persistence test base backed by SQL func NewTestBaseWithSQL(options *TestBaseOptions) *TestBase { - if options.DBName == "" { - options.DBName = "test_" + GenerateRandomDBName(3) - } logger := options.Logger if logger == nil { logger = log.NewTestLogger() diff --git a/common/persistence/persistence-tests/setup.go b/common/persistence/persistence-tests/setup.go index 6f16480ad5..1dd765ba6c 100644 --- a/common/persistence/persistence-tests/setup.go +++ b/common/persistence/persistence-tests/setup.go @@ -1,6 +1,10 @@ package persistencetests import ( + "fmt" + "os" + "path/filepath" + "go.temporal.io/server/common/config" "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql" "go.temporal.io/server/common/persistence/sql/sqlplugin/postgresql" @@ -9,6 +13,8 @@ import ( ) const ( + testCassandraSchemaDir = "schema/cassandra/" + testMySQLUser = "temporal" testMySQLPassword = "temporal" testMySQLSchemaDir = "schema/mysql/v8" @@ -24,10 +30,45 @@ const ( testSQLiteSchemaDir = "schema/sqlite/v3" // specify if mode is not "memory" ) +// GetTestClusterOption returns test options for the given store type and driver. +func GetTestClusterOption(storeType, driver string) *TestBaseOptions { + switch storeType { + case config.StoreTypeSQL: + switch driver { + case mysql.PluginName: + return GetMySQLTestClusterOption() + case postgresql.PluginName: + return GetPostgreSQLTestClusterOption() + case postgresql.PluginNamePGX: + return GetPostgreSQLPGXTestClusterOption() + case sqlite.PluginName: + return GetSQLiteMemoryTestClusterOption() + default: + panic(fmt.Sprintf("unknown sql driver: %v", driver)) + } + case config.StoreTypeNoSQL: + return GetCassandraTestClusterOption() + default: + panic(fmt.Sprintf("unknown store type: %v", storeType)) + } +} + +// GetCassandraTestClusterOption returns test options +func GetCassandraTestClusterOption() *TestBaseOptions { + return &TestBaseOptions{ + DBName: "test_" + GenerateRandomDBName(3), + DBHost: environment.GetCassandraAddress(), + DBPort: environment.GetCassandraPort(), + SchemaDir: testCassandraSchemaDir, + StoreType: config.StoreTypeNoSQL, + } +} + // GetMySQLTestClusterOption return test options func GetMySQLTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ SQLDBPluginName: mysql.PluginName, + DBName: "test_" + GenerateRandomDBName(3), DBUsername: testMySQLUser, DBPassword: testMySQLPassword, DBHost: environment.GetMySQLAddress(), @@ -41,6 +82,7 @@ func GetMySQLTestClusterOption() *TestBaseOptions { func GetPostgreSQLTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ SQLDBPluginName: postgresql.PluginName, + DBName: "test_" + GenerateRandomDBName(3), DBUsername: testPostgreSQLUser, DBPassword: testPostgreSQLPassword, DBHost: environment.GetPostgreSQLAddress(), @@ -54,6 +96,7 @@ func GetPostgreSQLTestClusterOption() *TestBaseOptions { func GetPostgreSQLPGXTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ SQLDBPluginName: postgresql.PluginNamePGX, + DBName: "test_" + GenerateRandomDBName(3), DBUsername: testPostgreSQLUser, DBPassword: testPostgreSQLPassword, DBHost: environment.GetPostgreSQLAddress(), @@ -66,14 +109,20 @@ func GetPostgreSQLPGXTestClusterOption() *TestBaseOptions { // GetSQLiteFileTestClusterOption return test options func GetSQLiteFileTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ - SQLDBPluginName: sqlite.PluginName, - DBUsername: testSQLiteUser, - DBPassword: testSQLitePassword, - DBHost: environment.GetLocalhostIP(), - DBPort: 0, - SchemaDir: testSQLiteSchemaDir, - StoreType: config.StoreTypeSQL, - ConnectAttributes: map[string]string{"cache": testSQLiteCache}, + SQLDBPluginName: sqlite.PluginName, + DBName: filepath.Join(os.TempDir(), "test_"+GenerateRandomDBName(3)), // put files in temp to avoid cluttering the project + DBUsername: testSQLiteUser, + DBPassword: testSQLitePassword, + DBHost: environment.GetLocalhostIP(), + DBPort: 0, + SchemaDir: testSQLiteSchemaDir, + StoreType: config.StoreTypeSQL, + ConnectAttributes: map[string]string{ + "cache": "shared", + "busy_timeout": "30000", + "journal_mode": "wal", + "synchronous": "normal", + }, } } @@ -81,6 +130,7 @@ func GetSQLiteFileTestClusterOption() *TestBaseOptions { func GetSQLiteMemoryTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ SQLDBPluginName: sqlite.PluginName, + DBName: "test_" + GenerateRandomDBName(3), DBUsername: testSQLiteUser, DBPassword: testSQLitePassword, DBHost: environment.GetLocalhostIP(), diff --git a/common/persistence/sql/test_sql_persistence.go b/common/persistence/sql/test_sql_persistence.go index 5a48793ba8..823dbd170f 100644 --- a/common/persistence/sql/test_sql_persistence.go +++ b/common/persistence/sql/test_sql_persistence.go @@ -143,10 +143,11 @@ func (s *TestCluster) DropDatabase() { if cfg2.PluginName == "sqlite" && cfg2.DatabaseName != ":memory:" && cfg2.ConnectAttributes["mode"] != "memory" { if len(cfg2.DatabaseName) > 3 { // 3 should mean not ., .., empty, or / - err := os.Remove(cfg2.DatabaseName) - if err != nil { - panic(err) - } + // Remove main database file + _ = os.Remove(cfg2.DatabaseName) + // Remove WAL mode files (may not exist if WAL wasn't used) + _ = os.Remove(cfg2.DatabaseName + "-wal") + _ = os.Remove(cfg2.DatabaseName + "-shm") } return } diff --git a/develop/github/docker-compose.yml b/develop/github/docker-compose.yml index 6e8dfa9ebe..add741ff62 100644 --- a/develop/github/docker-compose.yml +++ b/develop/github/docker-compose.yml @@ -7,6 +7,8 @@ services: CASSANDRA_LISTEN_ADDRESS: 127.0.0.1 MAX_HEAP_SIZE: "2G" HEAP_NEWSIZE: "200M" + # Increase native transport threads for handling more concurrent connections + JVM_EXTRA_OPTS: "-Dcassandra.native_transport_max_threads=512" mysql: image: mysql:8.0.29-oracle @@ -14,6 +16,7 @@ services: - "3306:3306" environment: MYSQL_ROOT_PASSWORD: root + command: --max-connections=500 volumes: - ./mysql-init:/docker-entrypoint-initdb.d @@ -24,6 +27,7 @@ services: environment: POSTGRES_USER: temporal POSTGRES_PASSWORD: temporal + command: postgres -c max_connections=500 volumes: - ./postgresql-init:/docker-entrypoint-initdb.d diff --git a/docs/development/testing.md b/docs/development/testing.md index 72b3d05780..195dbcdf2c 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -14,7 +14,8 @@ This document describes the project's testing setup, utilities and best practice - `TEMPORAL_TEST_LOG_FORMAT`: Controls the output format for test logs. Available options: `json` or `console` - `TEMPORAL_TEST_LOG_LEVEL`: Sets the verbosity level for test logging. Available levels: `debug`, `info`, `warn`, `error`, `fatal` - `TEMPORAL_TEST_OTEL_OUTPUT`: Enables OpenTelemetry (OTEL) trace output for failed tests to the provided file path. - +- `TEMPORAL_TEST_SHARED_CLUSTERS`: Number of shared clusters in the pool. Each can be used by multiple tests simultaneously. +- `TEMPORAL_TEST_DEDICATED_CLUSTERS`: Number of dedicated clusters in the pool. Each can be used by one test only at a time. ### Debugging via IDE #### GoLand diff --git a/tests/testcore/functional_test_base.go b/tests/testcore/functional_test_base.go index 1dbc580191..2106c293df 100644 --- a/tests/testcore/functional_test_base.go +++ b/tests/testcore/functional_test_base.go @@ -35,6 +35,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/persistence" + persistencetests "go.temporal.io/server/common/persistence/persistence-tests" "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/rpc" @@ -85,6 +86,11 @@ type ( // TODO (alex): replace with v2 taskPoller *taskpoller.TaskPoller + + // isShared indicates whether this cluster is shared between multiple tests. + // Certain operations (e.g. InjectHook, CloseShard) are not safe on shared clusters + // and will panic if called. + isShared bool } // TestClusterParams contains the variables which are used to configure test cluster via the TestClusterOption type. TestClusterParams struct { @@ -94,6 +100,7 @@ type ( EnableMTLS bool FaultInjectionConfig *config.FaultInjection NumHistoryShards int32 + SharedCluster bool } TestClusterOption func(params *TestClusterParams) ) @@ -156,6 +163,12 @@ func WithNumHistoryShards(n int32) TestClusterOption { } } +func WithSharedCluster() TestClusterOption { + return func(params *TestClusterParams) { + params.SharedCluster = true + } +} + func (s *FunctionalTestBase) GetTestCluster() *TestCluster { return s.testCluster } @@ -228,6 +241,12 @@ func (s *FunctionalTestBase) TearDownSuite() { } func (s *FunctionalTestBase) SetupSuiteWithCluster(options ...TestClusterOption) { + // Acquire a slot from the dedicated test cluster pool. + testClusterPool.dedicated.acquireSlot(s.T()) + s.setupCluster(options...) +} + +func (s *FunctionalTestBase) setupCluster(options ...TestClusterOption) { params := ApplyTestClusterOptions(options) // NOTE: A suite might set its own logger. Example: AcquireShardSuiteBase. @@ -254,6 +273,13 @@ func (s *FunctionalTestBase) SetupSuiteWithCluster(options ...TestClusterOption) EnableMTLS: params.EnableMTLS, } + // Apply configuration for shared clusters. + if params.SharedCluster { + // Use file-based SQLite for shared clusters to support parallel test access. + s.testClusterConfig.Persistence = *persistencetests.GetSQLiteFileTestClusterOption() + s.isShared = true + } + // Initialize the OTEL collector if OTEL is enabled. // Must be done before the test cluster is created, so that the collector can be used by the test cluster. if otelOutputDir := os.Getenv("TEMPORAL_TEST_OTEL_OUTPUT"); otelOutputDir != "" { @@ -568,17 +594,30 @@ func (s *FunctionalTestBase) DurationNear(value, target, tolerance time.Duration s.Less(value, target+tolerance) } -// Overrides one dynamic config setting for the duration of this test (or sub-test). The change -// will automatically be reverted at the end of the test (using t.Cleanup). The cleanup -// function is also returned if you want to revert the change before the end of the test. func (s *FunctionalTestBase) OverrideDynamicConfig(setting dynamicconfig.GenericSetting, value any) (cleanup func()) { return s.testCluster.host.overrideDynamicConfig(s.T(), setting.Key(), value) } func (s *FunctionalTestBase) InjectHook(key testhooks.Key, value any) (cleanup func()) { + if s.isShared { + s.T().Fatalf("InjectHook cannot be called on a shared cluster; use testcore.WithDedicatedCluster()") + } return s.testCluster.host.injectHook(s.T(), key, value) } +// CloseShard closes the shard that contains the given workflow. +// This is a cluster-global operation and cannot be called on shared clusters. +func (s *FunctionalTestBase) CloseShard(namespaceID string, workflowID string) { + if s.isShared { + s.T().Fatalf("CloseShard cannot be called on a shared cluster; use testcore.WithDedicatedCluster()") + } + shardID := common.WorkflowIDToHistoryShard(namespaceID, workflowID, s.testClusterConfig.HistoryConfig.NumHistoryShards) + _, err := s.AdminClient().CloseShard(NewContext(), &adminservice.CloseShardRequest{ + ShardId: shardID, + }) + s.Require().NoError(err) +} + func (s *FunctionalTestBase) GetNamespaceID(namespace string) string { namespaceResp, err := s.FrontendClient().DescribeNamespace(NewContext(), &workflowservice.DescribeNamespaceRequest{ Namespace: namespace, diff --git a/tests/testcore/test_cluster.go b/tests/testcore/test_cluster.go index 41eb5b8d7d..d8eeef4f73 100644 --- a/tests/testcore/test_cluster.go +++ b/tests/testcore/test_cluster.go @@ -35,9 +35,6 @@ import ( "go.temporal.io/server/common/namespace/nsreplication" "go.temporal.io/server/common/persistence" persistencetests "go.temporal.io/server/common/persistence/persistence-tests" - "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql" - "go.temporal.io/server/common/persistence/sql/sqlplugin/postgresql" - "go.temporal.io/server/common/persistence/sql/sqlplugin/sqlite" esclient "go.temporal.io/server/common/persistence/visibility/store/elasticsearch/client" "go.temporal.io/server/common/pprof" "go.temporal.io/server/common/primitives" @@ -144,37 +141,15 @@ type PersistenceTestBaseFactory interface { type defaultPersistenceTestBaseFactory struct{} +// GetPersistenceTestDefaults returns the default persistence options based on CLI flags. +// Use this when creating TestClusterConfig to ensure proper database configuration. +func GetPersistenceTestDefaults() persistencetests.TestBaseOptions { + return *persistencetests.GetTestClusterOption(cliFlags.persistenceType, cliFlags.persistenceDriver) +} + func (f *defaultPersistenceTestBaseFactory) NewTestBase(options *persistencetests.TestBaseOptions) *persistencetests.TestBase { - options.StoreType = cliFlags.persistenceType - switch cliFlags.persistenceType { - case config.StoreTypeSQL: - var ops *persistencetests.TestBaseOptions - switch cliFlags.persistenceDriver { - case mysql.PluginName: - ops = persistencetests.GetMySQLTestClusterOption() - case postgresql.PluginName: - ops = persistencetests.GetPostgreSQLTestClusterOption() - case postgresql.PluginNamePGX: - ops = persistencetests.GetPostgreSQLPGXTestClusterOption() - case sqlite.PluginName: - ops = persistencetests.GetSQLiteMemoryTestClusterOption() - default: - //nolint:forbidigo // test code - panic(fmt.Sprintf("unknown sql store driver: %v", cliFlags.persistenceDriver)) - } - options.SQLDBPluginName = cliFlags.persistenceDriver - options.DBUsername = ops.DBUsername - options.DBPassword = ops.DBPassword - options.DBHost = ops.DBHost - options.DBPort = ops.DBPort - options.SchemaDir = ops.SchemaDir - options.ConnectAttributes = ops.ConnectAttributes - case config.StoreTypeNoSQL: - // noop for now - default: - //nolint:forbidigo // test code - panic(fmt.Sprintf("unknown store type: %v", options.StoreType)) - } + defaults := GetPersistenceTestDefaults() + options.ApplyDefaults(&defaults) if cliFlags.enableFaultInjection != "" && options.FaultInjection == nil { // If -enableFaultInjection is passed to the test runner, then default fault injection config is added to the persistence options. diff --git a/tests/testcore/test_cluster_pool.go b/tests/testcore/test_cluster_pool.go new file mode 100644 index 0000000000..28fa0dd846 --- /dev/null +++ b/tests/testcore/test_cluster_pool.go @@ -0,0 +1,189 @@ +package testcore + +import ( + "os" + "runtime" + "strconv" + "sync" + "sync/atomic" + "testing" + + "go.temporal.io/server/common/dynamicconfig" +) + +var testClusterPool *clusterPool + +func init() { + sharedSize := max(1, runtime.GOMAXPROCS(0)/2) + if v := os.Getenv("TEMPORAL_TEST_SHARED_CLUSTERS"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + panic("TEMPORAL_TEST_SHARED_CLUSTERS must be a positive integer") + } + sharedSize = n + } + + dedicatedSize := runtime.GOMAXPROCS(0) + if v := os.Getenv("TEMPORAL_TEST_DEDICATED_CLUSTERS"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + panic("TEMPORAL_TEST_DEDICATED_CLUSTERS must be a positive integer") + } + dedicatedSize = n + } + + // In CI, recreate clusters after 50 tests to prevent resource accumulation. + // Locally, clusters are reused indefinitely for faster iteration. + var maxUsage int + if os.Getenv("CI") != "" { + maxUsage = 50 + } + + sharedPool := newPool(sharedSize, false) + sharedPool.maxUsage = maxUsage + + dedicatedPool := newPool(dedicatedSize, true) + dedicatedPool.maxUsage = maxUsage + + testClusterPool = &clusterPool{ + shared: sharedPool, + dedicated: dedicatedPool, + } +} + +// pool manages a fixed number of test clusters with lazy initialization. +type pool struct { + clusters []*FunctionalTestBase + inits []sync.Once + counter atomic.Int64 // for round-robin (when slots is nil) + slots chan int // for exclusive access (nil means shared/concurrent access) + + // For shared pools: track usage and support teardown/recreate after maxUsage tests + usageCounts []atomic.Int64 + clusterMu []sync.Mutex // protects cluster teardown/recreate + maxUsage int // max tests per cluster before recreate (0 = unlimited) + createFn func() *FunctionalTestBase +} + +func newPool(size int, exclusive bool) *pool { + p := &pool{ + clusters: make([]*FunctionalTestBase, size), + inits: make([]sync.Once, size), + usageCounts: make([]atomic.Int64, size), + clusterMu: make([]sync.Mutex, size), + } + if exclusive { + p.slots = make(chan int, size) + for i := range size { + p.slots <- i + } + } + return p +} + +// get returns a cluster from the pool, creating it lazily if needed. +// For exclusive pools, blocks until a slot is available and registers cleanup. +// For shared pools, uses round-robin. +// Both pool types may recreate clusters after maxUsage tests (in CI). +func (p *pool) get(t *testing.T, createCluster func() *FunctionalTestBase) *FunctionalTestBase { + var idx int + if p.slots != nil { + idx = <-p.slots + t.Cleanup(func() { p.slots <- idx }) + } else { + idx = int(p.counter.Add(1)-1) % len(p.clusters) + } + + // Check if we need to recreate the cluster after maxUsage tests + if p.maxUsage > 0 { + usage := p.usageCounts[idx].Add(1) + if usage > int64(p.maxUsage) { + p.clusterMu[idx].Lock() + // Double-check after acquiring lock + if p.usageCounts[idx].Load() > int64(p.maxUsage) && p.clusters[idx] != nil { + if err := p.clusters[idx].testCluster.TearDownCluster(); err != nil { + t.Logf("Failed to tear down cluster %d: %v", idx, err) + } + p.clusters[idx] = createCluster() + p.usageCounts[idx].Store(1) // Reset to 1 (this test counts) + } + p.clusterMu[idx].Unlock() + } + } + + // Lazy initialization for first use + p.inits[idx].Do(func() { + p.clusters[idx] = createCluster() + }) + + cluster := p.clusters[idx] + cluster.SetT(t) + return cluster +} + +// acquireSlot gets exclusive access to a slot without using a pooled cluster. +// Used when a fresh cluster is needed (e.g., custom dynamic config). +func (p *pool) acquireSlot(t *testing.T) { + if p.slots == nil { + return + } + idx := <-p.slots + t.Cleanup(func() { p.slots <- idx }) +} + +type clusterPool struct { + shared *pool + dedicated *pool +} + +func (p *clusterPool) get(t *testing.T, dedicated bool, dynamicConfig map[dynamicconfig.Key]any) *FunctionalTestBase { + if dedicated || len(dynamicConfig) > 0 { + return p.getDedicated(t, dynamicConfig) + } + return p.getShared(t) +} + +func (p *clusterPool) getShared(t *testing.T) *FunctionalTestBase { + return p.shared.get(t, func() *FunctionalTestBase { + return p.createCluster(t, nil, true) + }) +} + +func (p *clusterPool) getDedicated(t *testing.T, dynamicConfig map[dynamicconfig.Key]any) *FunctionalTestBase { + if len(dynamicConfig) > 0 { + // Custom dynamic config requires a fresh cluster (can't reuse). + p.dedicated.acquireSlot(t) + cluster := p.createCluster(t, dynamicConfig, false) + + // Register cleanup to tear down the cluster when the test completes. + t.Cleanup(func() { + if err := cluster.testCluster.TearDownCluster(); err != nil { + t.Logf("Failed to tear down cluster: %v", err) + } + }) + + return cluster + } + + // If no custom dynamic config is provided, reuse an existing cluster. + return p.dedicated.get(t, func() *FunctionalTestBase { + return p.createCluster(t, nil, false) + }) +} + +func (p *clusterPool) createCluster(t *testing.T, dynamicConfig map[dynamicconfig.Key]any, shared bool) *FunctionalTestBase { + tbase := &FunctionalTestBase{} + tbase.SetT(t) + + var opts []TestClusterOption + if shared { + opts = append(opts, WithSharedCluster()) + } + if len(dynamicConfig) > 0 { + opts = append(opts, WithDynamicConfigOverrides(dynamicConfig)) + } + + tbase.setupCluster(opts...) + + return tbase +} diff --git a/tests/testcore/test_env.go b/tests/testcore/test_env.go new file mode 100644 index 0000000000..69eb7d1a00 --- /dev/null +++ b/tests/testcore/test_env.go @@ -0,0 +1,188 @@ +package testcore + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/historyrequire" + "go.temporal.io/server/common/testing/taskpoller" + "go.temporal.io/server/common/testing/testvars" +) + +var _ Env = (*testEnv)(nil) + +type Env interface { + T() *testing.T + Namespace() namespace.Name + FrontendClient() workflowservice.WorkflowServiceClient + GetTestCluster() *TestCluster + CloseShard(namespaceID string, workflowID string) + OverrideDynamicConfig(setting dynamicconfig.GenericSetting, value any) (cleanup func()) +} + +type testEnv struct { + *FunctionalTestBase + *require.Assertions + historyrequire.HistoryRequire + + Logger log.Logger + + cluster *TestCluster + nsName namespace.Name + taskPoller *taskpoller.TaskPoller + t *testing.T + tv *testvars.TestVars +} + +type TestOption func(*testOptions) + +type testOptions struct { + dedicatedCluster bool + dynamicConfigSettings []dynamicConfigOverride +} + +type dynamicConfigOverride struct { + setting dynamicconfig.GenericSetting + value any +} + +// WithDedicatedCluster requests a dedicated (non-shared) cluster for the test. +// Use this for tests that have cluster-global side effects. +func WithDedicatedCluster() TestOption { + return func(o *testOptions) { + o.dedicatedCluster = true + } +} + +// WithDynamicConfig overrides a dynamic config setting for the test. +// For settings that can be namespace-scoped, a namespace constraint is applied. +// For all others that require a dedicated cluster, this implies `WithDedicatedCluster`. +func WithDynamicConfig(setting dynamicconfig.GenericSetting, value any) TestOption { + return func(o *testOptions) { + if err := setting.Validate(value); err != nil { + panic(fmt.Sprintf("invalid value for setting %s: %v", setting.Key(), err)) + } + if !canBeNamespaceScoped(setting.Precedence()) { + o.dedicatedCluster = true + } + o.dynamicConfigSettings = append(o.dynamicConfigSettings, dynamicConfigOverride{setting: setting, value: value}) + } +} + +// NewEnv creates a new test environment with access to a Temporal cluster. +// The test is automatically marked as parallel. +func NewEnv(t *testing.T, opts ...TestOption) *testEnv { + t.Parallel() + + var options testOptions + for _, opt := range opts { + opt(&options) + } + + // For dedicated clusters, pass all dynamic config settings at cluster creation. + var startupConfig map[dynamicconfig.Key]any + if options.dedicatedCluster && len(options.dynamicConfigSettings) > 0 { + startupConfig = make(map[dynamicconfig.Key]any, len(options.dynamicConfigSettings)) + for _, override := range options.dynamicConfigSettings { + startupConfig[override.setting.Key()] = override.value + } + } + + base := testClusterPool.get(t, options.dedicatedCluster, startupConfig) + cluster := base.GetTestCluster() + + // Create a dedicated namespace for the test to help with test isolation. + ns := namespace.Name(RandomizeStr(t.Name())) + if _, err := base.RegisterNamespace( + ns, + 1, // 1 day retention + enumspb.ARCHIVAL_STATE_DISABLED, + "", + "", + ); err != nil { + t.Fatalf("Failed to register namespace: %v", err) + } + + env := &testEnv{ + FunctionalTestBase: base, + Assertions: require.New(t), + HistoryRequire: historyrequire.New(t), + cluster: cluster, + nsName: ns, + Logger: base.Logger, + taskPoller: taskpoller.New(t, cluster.FrontendClient(), ns.String()), + t: t, + tv: testvars.New(t), + } + + // For shared clusters, apply all dynamic config settings as overrides. + if !options.dedicatedCluster && len(options.dynamicConfigSettings) > 0 { + for _, override := range options.dynamicConfigSettings { + env.OverrideDynamicConfig(override.setting, override.value) + } + } + + return env +} + +// Use test env-specific namespace here for test isolation. +func (e *testEnv) Namespace() namespace.Name { + return e.nsName +} + +func (e *testEnv) TaskPoller() *taskpoller.TaskPoller { + return e.taskPoller +} + +func (e *testEnv) T() *testing.T { + return e.t +} + +func (e *testEnv) Tv() *testvars.TestVars { + return e.tv +} + +// OverrideDynamicConfig overrides a dynamic config setting for the duration of this test. +// For settings that can be namespace-scoped, a namespace constraint is applied. +// All others cannot be applied to a shared cluster and require `WithDedicatedCluster`. +func (e *testEnv) OverrideDynamicConfig(setting dynamicconfig.GenericSetting, value any) (cleanup func()) { + if e.isShared { + if !canBeNamespaceScoped(setting.Precedence()) { + e.t.Fatalf("OverrideDynamicConfig for setting %s (precedence %v) cannot be called on a shared cluster; use testcore.WithDedicatedCluster()", setting.Key(), setting.Precedence()) + } + + // Wrap value with namespace constraint for test isolation on shared clusters. + ns := e.nsName.String() + if cvs, ok := value.([]dynamicconfig.ConstrainedValue); ok { + result := make([]dynamicconfig.ConstrainedValue, len(cvs)) + for i, cv := range cvs { + cv.Constraints.Namespace = ns + result[i] = cv + } + value = result + } else { + value = []dynamicconfig.ConstrainedValue{{ + Constraints: dynamicconfig.Constraints{Namespace: ns}, + Value: value, + }} + } + } + return e.cluster.host.overrideDynamicConfig(e.t, setting.Key(), value) +} + +func canBeNamespaceScoped(p dynamicconfig.Precedence) bool { + switch p { + case dynamicconfig.PrecedenceNamespace, + dynamicconfig.PrecedenceTaskQueue, + dynamicconfig.PrecedenceDestination: + return true + default: + return false + } +} diff --git a/tests/update_workflow_test.go b/tests/update_workflow_test.go index 7ab820da11..70059cd164 100644 --- a/tests/update_workflow_test.go +++ b/tests/update_workflow_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/stretchr/testify/suite" + "github.com/stretchr/testify/assert" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -33,31 +33,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" ) -type UpdateWorkflowSuite struct { - testcore.FunctionalTestBase -} - -func TestUpdateWorkflowSuite(t *testing.T) { - t.Parallel() - s := new(UpdateWorkflowSuite) - suite.Run(t, s) -} - -// Simulating a graceful shard closure. The shard finalizer will clear the workflow context, -// any update requests are aborted and the frontend retries any in-flight update requests. -func (s *UpdateWorkflowSuite) clearUpdateRegistryAndAbortPendingUpdates(tv *testvars.TestVars) { - s.closeShard(tv.WorkflowID()) -} - -// Simulating an unexpected loss of the update registry due to a crash. The shard finalizer won't run, -// therefore the workflow context is NOT cleared, pending update requests are NOT aborted and will time out. -func (s *UpdateWorkflowSuite) loseUpdateRegistryAndAbandonPendingUpdates(tv *testvars.TestVars) { - cleanup := s.OverrideDynamicConfig(dynamicconfig.ShardFinalizerTimeout, 0) - defer cleanup() - s.closeShard(tv.WorkflowID()) -} - -func (s *UpdateWorkflowSuite) speculativeWorkflowTaskOutcomes( +func speculativeWorkflowTaskOutcomes( snap map[string][]*metricstest.CapturedRecording, ) (commits, rollbacks int) { for range snap[metrics.SpeculativeWorkflowTaskCommits.Name()] { @@ -69,39 +45,68 @@ func (s *UpdateWorkflowSuite) speculativeWorkflowTaskOutcomes( return } -func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_AcceptComplete() { - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, +func clearUpdateRegistryAndAbortPendingUpdates(s testcore.Env, tv *testvars.TestVars) { + closeShard(s, tv.WorkflowID()) +} + +func loseUpdateRegistryAndAbandonPendingUpdates(s testcore.Env, tv *testvars.TestVars) { + cleanup := s.OverrideDynamicConfig(dynamicconfig.ShardFinalizerTimeout, 0) + defer cleanup() + closeShard(s, tv.WorkflowID()) +} + +func closeShard(s testcore.Env, wid string) { + s.T().Helper() + + resp, err := s.FrontendClient().DescribeNamespace(testcore.NewContext(), &workflowservice.DescribeNamespaceRequest{ + Namespace: s.Namespace().String(), + }) + if err != nil { + s.T().Fatalf("Failed to describe namespace: %v", err) } - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) + s.CloseShard(resp.NamespaceInfo.Id, wid) +} - runID := mustStartWorkflow(s, tv) +func TestWorkflowUpdateSuite(t *testing.T) { + t.Run("EmptySpeculativeWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } - capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() - defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Uses CaptureMetricsHandler which requires a dedicated cluster to avoid metric interference. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + runID := mustStartWorkflow(s, s.Tv()) - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } + + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -109,130 +114,134 @@ func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_AcceptComplete() 5 WorkflowTaskScheduled // Speculative WT events are not written to the history yet. 6 WorkflowTaskStarted `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) - - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res.NewTask) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) - - // Test non-blocking poll - for _, waitPolicy := range []*updatepb.WaitPolicy{{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_UNSPECIFIED}, nil} { - pollUpdateResp, err := pollUpdate(s, tv, waitPolicy) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() s.NoError(err) - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, pollUpdateResp.Stage) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), pollUpdateResp.Outcome.GetSuccess())) - // Even if tv doesn't have RunID, it should be returned as part of UpdateRef. - s.Equal(runID, pollUpdateResp.UpdateRef.GetWorkflowExecution().RunId) - } - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + updateResultCh := sendUpdateNoError(s, tv) - commits, rollbacks := s.speculativeWorkflowTaskOutcomes(capture.Snapshot()) - s.Equal(1, commits) - s.Equal(0, rollbacks) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res.NewTask) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - events := s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution()) + // Test non-blocking poll + for _, waitPolicy := range []*updatepb.WaitPolicy{{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_UNSPECIFIED}, nil} { + pollUpdateResp, err := pollUpdate(s, s.Tv(), waitPolicy) + s.NoError(err) + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, pollUpdateResp.Stage) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), pollUpdateResp.Outcome.GetSuccess())) + // Even if tv doesn't have RunID, it should be returned as part of UpdateRef. + s.Equal(runID, pollUpdateResp.UpdateRef.GetWorkflowExecution().RunId) + } - s.EqualHistoryEvents(` + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + commits, rollbacks := speculativeWorkflowTaskOutcomes(capture.Snapshot()) + s.Equal(1, commits) + s.Equal(0, rollbacks) + + events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Was speculative WT... 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted // ...and events were written to the history when WT completes. + 7 WorkflowTaskCompleted // ...and events were written to the history when WT completes. 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} `, events) - }) - } -} + }) + } + }) -func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_AcceptComplete() { - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, - } + t.Run("NotEmptySpeculativeWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } - runID := mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with update unrelated command. - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv.ActivityID(), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: tv.Any().InfiniteTimeout(), - }}, - }}, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with update unrelated command. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: s.Tv().ActivityID(), + ActivityType: s.Tv().ActivityType(), + TaskQueue: s.Tv().TaskQueue(), + ScheduleToCloseTimeout: s.Tv().Any().InfiniteTimeout(), + }}, + }}, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -241,65 +250,66 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_AcceptComplete 6 WorkflowTaskScheduled // Speculative WFT with ActivityTaskScheduled(5) event after WorkflowTaskCompleted(4). 7 WorkflowTaskStarted `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(6, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(6, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) + updateResultCh := sendUpdateNoError(s, tv) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -307,100 +317,103 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_AcceptComplete 5 ActivityTaskScheduled 6 WorkflowTaskScheduled // Speculative WFT was persisted when completed (event 8) 7 WorkflowTaskStarted - 8 WorkflowTaskCompleted + 8 WorkflowTaskCompleted 9 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 6} // WTScheduled event which delivered update to the worker. 10 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 9} `, events) - }) - } -} + }) + } + }) -func (s *UpdateWorkflowSuite) TestFirstNormalScheduledWorkflowTask_AcceptComplete() { + t.Run("FirstNormalScheduledWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, - } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) - - runID := mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted // First normal WT. No speculative WT was created. `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(2, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(2, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - updateResultCh := sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) + updateResultCh := sendUpdateNoError(s, tv) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(1, wtHandlerCalls) - s.Equal(1, msgHandlerCalls) + s.Equal(1, wtHandlerCalls) + s.Equal(1, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -408,41 +421,43 @@ func (s *UpdateWorkflowSuite) TestFirstNormalScheduledWorkflowTask_AcceptComplet 5 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 2} // WTScheduled event which delivered update to the worker. 6 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 5} `, events) - }) - } -} + }) + } + }) -func (s *UpdateWorkflowSuite) TestNormalScheduledWorkflowTask_AcceptComplete() { + t.Run("NormalScheduledWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, - } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) - - runID := mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -450,71 +465,72 @@ func (s *UpdateWorkflowSuite) TestNormalScheduledWorkflowTask_AcceptComplete() { 5 WorkflowExecutionSignaled 6 WorkflowTaskScheduled // This WT was already created by signal and no speculative WT was created. 7 WorkflowTaskStarted`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Require().True(len(task.Messages) > 0, "Task has no messages", task) - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Require().NotEmpty(task.Messages, "Task has no messages", task) + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(6, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(6, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - // Send signal to schedule new WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) + // Send signal to schedule new WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - updateResultCh := sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) + updateResultCh := sendUpdateNoError(s, tv) - // Process update in workflow. It will be attached to existing WT. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) + // Process update in workflow. It will be attached to existing WT. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -526,38 +542,38 @@ func (s *UpdateWorkflowSuite) TestNormalScheduledWorkflowTask_AcceptComplete() { 9 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 6} // WTScheduled event which delivered update to the worker. 10 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 9} `, events) - }) - } -} + }) + } + }) -func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewEmptySpeculativeWorkflowTask_Rejected() { - tv := testvars.New(s.T()) + t.Run("RunningWorkflowTask_NewEmptySpeculativeWorkflowTask_Rejected", func(t *testing.T) { + // Uses CaptureMetricsHandler which requires a dedicated cluster to avoid metric interference. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() - defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) + var updateResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - var updateResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Send update after 1st WT has started. - updateResultCh = sendUpdateNoError(s, tv) - // Completes WT with empty command list to create next WFT w/o events. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Send update after 1st WT has started. + updateResultCh = sendUpdateNoError(s, s.Tv()) + // Completes WT with empty command list to create next WFT w/o events. + return nil, nil + case 2: + s.EqualHistory(` 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Speculative WT2 which was created while completing WT1. 6 WorkflowTaskStarted`, task.History) - // Message handler rejects update. - return nil, nil - case 3: - s.EqualHistory(` + // Message handler rejects update. + return nil, nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -565,79 +581,78 @@ func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewEmptySpeculativeWorkflo 5 WorkflowExecutionSignaled 6 WorkflowTaskScheduled 7 WorkflowTaskStarted`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1) - updRequestMsg := task.Messages[0] - s.EqualValues(5, updRequestMsg.GetEventId()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1, 3: + return nil, nil + case 2: + s.Len(task.Messages, 1) + updRequestMsg := task.Messages[0] + s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - case 3: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT which starts 1st update. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - wt1Resp := res.NewTask + // Drain first WT which starts 1st update. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + wt1Resp := res.NewTask - // Reject update in 2nd WT. - wt2Resp, err := poller.HandlePartialWorkflowTask(wt1Resp.GetWorkflowTask(), false) - s.NoError(err) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(3, wt2Resp.ResetHistoryEventId) + // Reject update in 2nd WT. + wt2Resp, err := poller.HandlePartialWorkflowTask(wt1Resp.GetWorkflowTask(), false) + s.NoError(err) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(3, wt2Resp.ResetHistoryEventId) - // Send signal to create WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) + // Send signal to create WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(completeWorkflowResp) + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(completeWorkflowResp) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - commits, rollbacks := s.speculativeWorkflowTaskOutcomes(capture.Snapshot()) - s.Equal(0, commits) - s.Equal(1, rollbacks) + commits, rollbacks := speculativeWorkflowTaskOutcomes(capture.Snapshot()) + s.Equal(0, commits) + s.Equal(1, rollbacks) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -647,43 +662,42 @@ func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewEmptySpeculativeWorkflo 7 WorkflowTaskStarted 8 WorkflowTaskCompleted 9 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewNotEmptySpeculativeWorkflowTask_Rejected() { + t.Run("RunningWorkflowTask_NewNotEmptySpeculativeWorkflowTask_Rejected", func(t *testing.T) { + s := testcore.NewEnv(t) - tv := testvars.New(s.T()) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + var updateResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - var updateResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Send update after 1st WT has started. - updateResultCh = sendUpdateNoError(s, tv) - // Completes WT with update unrelated commands to create events that will be in the next speculative WFT. - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv.ActivityID(), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: tv.Any().InfiniteTimeout(), - }}, - }}, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Send update after 1st WT has started. + updateResultCh = sendUpdateNoError(s, s.Tv()) + // Completes WT with update unrelated commands to create events that will be in the next speculative WFT. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: s.Tv().ActivityID(), + ActivityType: s.Tv().ActivityType(), + TaskQueue: s.Tv().TaskQueue(), + ScheduleToCloseTimeout: s.Tv().Any().InfiniteTimeout(), + }}, + }}, nil + case 2: + s.EqualHistory(` 4 WorkflowTaskCompleted 5 ActivityTaskScheduled 6 WorkflowTaskScheduled // Speculative WFT2 with event (5) which was created while completing WFT1. 7 WorkflowTaskStarted`, task.History) - // Message handler rejects update. - return nil, nil - case 3: - s.EqualHistory(` + // Message handler rejects update. + return nil, nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -698,81 +712,80 @@ func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewNotEmptySpeculativeWork 12 WorkflowTaskStarted `, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1) - updRequestMsg := task.Messages[0] - s.EqualValues(6, updRequestMsg.GetEventId()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1, 3: + return nil, nil + case 2: + s.Len(task.Messages, 1) + updRequestMsg := task.Messages[0] + s.EqualValues(6, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - case 3: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT which starts 1st update. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - wt1Resp := res.NewTask + // Drain first WT which starts 1st update. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + wt1Resp := res.NewTask - // Reject update in 2nd WT. - wt2Resp, err := poller.HandlePartialWorkflowTask(wt1Resp.GetWorkflowTask(), false) - s.NoError(err) - s.NotNil(wt2Resp) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, wt2Resp.ResetHistoryEventId) + // Reject update in 2nd WT. + wt2Resp, err := poller.HandlePartialWorkflowTask(wt1Resp.GetWorkflowTask(), false) + s.NoError(err) + s.NotNil(wt2Resp) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, wt2Resp.ResetHistoryEventId) - // Schedule new WFT. - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + // Schedule new WFT. + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(completeWorkflowResp) - s.EqualValues(0, completeWorkflowResp.NewTask.ResetHistoryEventId) + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(completeWorkflowResp) + s.EqualValues(0, completeWorkflowResp.NewTask.ResetHistoryEventId) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -787,574 +800,582 @@ func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewNotEmptySpeculativeWork 12 WorkflowTaskStarted 13 WorkflowTaskCompleted 14 WorkflowExecutionCompleted`, events) -} - -func (s *UpdateWorkflowSuite) TestCompletedWorkflow() { - s.Run("receive outcome from completed Update", func() { - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - res := s.UpdateAcceptCompleteCommands(tv) - res = append(res, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }) - return res, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - return s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - - // Send Update request. - updateResultCh := sendUpdateNoError(s, tv) - - // Complete Update and Workflow. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - - // Receive Update result. - updateResult1 := <-updateResultCh - s.NotNil(updateResult1.GetOutcome().GetSuccess()) - - // Send same Update request again, receiving the same Update result. - updateResultCh = sendUpdateNoError(s, tv) - updateResult2 := <-updateResultCh - s.EqualValues(updateResult1, updateResult2) }) - s.Run("receive update failure from accepted Update", func() { - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - res := s.UpdateAcceptCommands(tv) - res = append(res, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }) - return res, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - return s.UpdateAcceptMessages(tv, task.Messages[0]), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - - // Send Update request. - updateResultCh := sendUpdate(testcore.NewContext(), s, tv) - - // Accept Update and complete Workflow. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - - // Receive Update result. - updateResult1 := <-updateResultCh - s.NoError(updateResult1.err) - s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult1.response.GetOutcome().GetFailure().GetMessage()) - - // Send same Update request again, receiving the same failure. - updateResultCh = sendUpdate(testcore.NewContext(), s, tv) - updateResult2 := <-updateResultCh - s.NoError(updateResult2.err) - s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult2.response.GetOutcome().GetFailure().GetMessage()) - }) -} - -func (s *UpdateWorkflowSuite) TestValidateWorkerMessages() { - testCases := []struct { - Name string - RespondWorkflowTaskError string - MessageFn func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message - CommandFn func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command - }{ - { - Name: "message-update-id-not-found-and-accepted-request-not-set", - RespondWorkflowTaskError: "wasn't found", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: tv.UpdateID() + tv.Any().String(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: nil, // Important not to pass original request back. - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - } - }, - }, - { - Name: "message-update-id-not-found-and-accepted-request-is-set", - RespondWorkflowTaskError: "", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: tv.UpdateID() + tv.Any().String(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, // Update will be resurrected from original request. - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - } - }, - }, - { - Name: "command-reference-missed-message", - RespondWorkflowTaskError: "referenced absent message ID", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.Any().String(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - } - }, - }, - { - Name: "complete-without-accept", - RespondWorkflowTaskError: "invalid state transition attempted", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-completed", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Response{ - Meta: updRequest.GetMeta(), - Outcome: &updatepb.Outcome{ - Value: &updatepb.Outcome_Success{ - Success: tv.Any().Payloads(), - }, - }, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-completed", - }}, - }, - } - }, - }, - { - Name: "accept-twice", - RespondWorkflowTaskError: "invalid state transition attempted", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.WithMessageIDNumber(1).MessageID(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - { - Id: tv.WithMessageIDNumber(2).MessageID(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.WithMessageIDNumber(1).MessageID(), - }}, - }, - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.WithMessageIDNumber(2).MessageID(), - }}, - }, - } - }, - }, - { - Name: "success-case", - RespondWorkflowTaskError: "", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - { - Id: tv.MessageID() + "_update-completed", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Response{ - Meta: updRequest.GetMeta(), - Outcome: &updatepb.Outcome{ - Value: &updatepb.Outcome_Success{ - Success: tv.Any().Payloads(), - }, - }, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-completed", - }}, - }, - } - }, - }, - { - Name: "success-case-no-commands", // PROTOCOL_MESSAGE commands are optional. - RespondWorkflowTaskError: "", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.Any().String(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - { - Id: tv.Any().String(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Response{ - Meta: updRequest.GetMeta(), - Outcome: &updatepb.Outcome{ - Value: &updatepb.Outcome_Success{ - Success: tv.Any().Payloads(), - }, - }, - }), - }, - } - }, - }, - { - Name: "invalid-command-order", - RespondWorkflowTaskError: "invalid state transition attempted", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - { - Id: tv.MessageID() + "_update-completed", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Response{ - Meta: updRequest.GetMeta(), - Outcome: &updatepb.Outcome{ - Value: &updatepb.Outcome_Success{ - Success: tv.Any().Payloads(), - }, - }, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - // Complete command goes before Accept command. - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-completed", - }}, - }, - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - } - }, - }, - } - - for _, tc := range testCases { - s.Run(tc.Name, func() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) + t.Run("CompletedWorkflow", func(t *testing.T) { + t.Run("receive outcome from completed Update", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + wtHandlerCalls := 0 wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - if tc.CommandFn == nil { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + res := s.UpdateAcceptCompleteCommands(s.Tv()) + res = append(res, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }) + return res, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) return nil, nil } - return tc.CommandFn(tv, task.History), nil } + msgHandlerCalls := 0 msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - if tc.MessageFn == nil { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + return s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) return nil, nil } - updRequestMsg := task.Messages[0] - return tc.MessageFn(tv, updRequestMsg), nil } + //nolint:staticcheck // SA1019 TaskPoller replacement needed poller := &testcore.TaskPoller{ Client: s.FrontendClient(), Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), WorkflowTaskHandler: wtHandler, MessageHandler: msgHandler, Logger: s.Logger, T: s.T(), } - halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) - defer cancel() - updateResultCh := sendUpdate(halfSecondTimeoutCtx, s, tv) - - // Process update in workflow. - _, err := poller.PollAndProcessWorkflowTask() - updateResult := <-updateResultCh - if tc.RespondWorkflowTaskError != "" { - s.Error(err, "RespondWorkflowTaskCompleted should return an error contains `%v`", tc.RespondWorkflowTaskError) - s.Contains(err.Error(), tc.RespondWorkflowTaskError) - - var wfNotReady *serviceerror.WorkflowNotReady - s.ErrorAs(updateResult.err, &wfNotReady, "API caller should get serviceerror.WorkflowNotReady, if server got a validation error while processing worker response.") - s.Contains(updateResult.err.Error(), "Unable to perform workflow execution update due to unexpected workflow task failure.") - s.Nil(updateResult.response) - } else { - s.NoError(err) - s.NoError(updateResult.err) - } - }) - } -} - -func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete() { - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, - } - - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) - - runID := mustStartWorkflow(s, tv) - - // Drain existing first WT from regular task queue, but respond with sticky queue enabled response, next WT will go to sticky queue. - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - StickyAttributes: tv.StickyExecutionAttributes(3 * time.Second), - }, nil - }) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) s.NoError(err) - go func() { - // Process update in workflow task (it is sticky). - res, err := s.TaskPoller(). - PollWorkflowTask(&workflowservice.PollWorkflowTaskQueueRequest{TaskQueue: tv.StickyTaskQueue()}). - HandleTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - // This WT contains partial history because sticky was enabled. - s.EqualHistory(` + // Send Update request. + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Complete Update and Workflow. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + + // Receive Update result. + updateResult1 := <-updateResultCh + s.NotNil(updateResult1.GetOutcome().GetSuccess()) + + // Send same Update request again, receiving the same Update result. + updateResultCh = sendUpdateNoError(s, s.Tv()) + updateResult2 := <-updateResultCh + s.Equal(updateResult1, updateResult2) + }) + + t.Run("receive update failure from accepted Update", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + res := s.UpdateAcceptCommands(s.Tv()) + res = append(res, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }) + return res, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + return s.UpdateAcceptMessages(s.Tv(), task.Messages[0]), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + + // Send Update request. + updateResultCh := sendUpdate(testcore.NewContext(), s, s.Tv()) + + // Accept Update and complete Workflow. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + + // Receive Update result. + updateResult1 := <-updateResultCh + s.NoError(updateResult1.err) + s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult1.response.GetOutcome().GetFailure().GetMessage()) + + // Send same Update request again, receiving the same failure. + updateResultCh = sendUpdate(testcore.NewContext(), s, s.Tv()) + updateResult2 := <-updateResultCh + s.NoError(updateResult2.err) + s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult2.response.GetOutcome().GetFailure().GetMessage()) + }) + }) + + t.Run("ValidateWorkerMessages", func(t *testing.T) { + testCases := []struct { + Name string + RespondWorkflowTaskError string + MessageFn func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message + CommandFn func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command + }{ + { + Name: "message-update-id-not-found-and-accepted-request-not-set", + RespondWorkflowTaskError: "wasn't found", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-accepted", + ProtocolInstanceId: tv.UpdateID() + tv.Any().String(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: nil, // Important not to pass original request back. + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + } + }, + }, + { + Name: "message-update-id-not-found-and-accepted-request-is-set", + RespondWorkflowTaskError: "", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-accepted", + ProtocolInstanceId: tv.UpdateID() + tv.Any().String(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, // Update will be resurrected from original request. + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + } + }, + }, + { + Name: "command-reference-missed-message", + RespondWorkflowTaskError: "referenced absent message ID", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.Any().String(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + } + }, + }, + { + Name: "complete-without-accept", + RespondWorkflowTaskError: "invalid state transition attempted", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-completed", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Response{ + Meta: updRequest.GetMeta(), + Outcome: &updatepb.Outcome{ + Value: &updatepb.Outcome_Success{ + Success: tv.Any().Payloads(), + }, + }, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-completed", + }}, + }, + } + }, + }, + { + Name: "accept-twice", + RespondWorkflowTaskError: "invalid state transition attempted", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.WithMessageIDNumber(1).MessageID(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + { + Id: tv.WithMessageIDNumber(2).MessageID(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.WithMessageIDNumber(1).MessageID(), + }}, + }, + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.WithMessageIDNumber(2).MessageID(), + }}, + }, + } + }, + }, + { + Name: "success-case", + RespondWorkflowTaskError: "", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-accepted", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + { + Id: tv.MessageID() + "_update-completed", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Response{ + Meta: updRequest.GetMeta(), + Outcome: &updatepb.Outcome{ + Value: &updatepb.Outcome_Success{ + Success: tv.Any().Payloads(), + }, + }, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-completed", + }}, + }, + } + }, + }, + { + Name: "success-case-no-commands", // PROTOCOL_MESSAGE commands are optional. + RespondWorkflowTaskError: "", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.Any().String(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + { + Id: tv.Any().String(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Response{ + Meta: updRequest.GetMeta(), + Outcome: &updatepb.Outcome{ + Value: &updatepb.Outcome_Success{ + Success: tv.Any().Payloads(), + }, + }, + }), + }, + } + }, + }, + { + Name: "invalid-command-order", + RespondWorkflowTaskError: "invalid state transition attempted", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-accepted", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + { + Id: tv.MessageID() + "_update-completed", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Response{ + Meta: updRequest.GetMeta(), + Outcome: &updatepb.Outcome{ + Value: &updatepb.Outcome_Success{ + Success: tv.Any().Payloads(), + }, + }, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + // Complete command goes before Accept command. + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-completed", + }}, + }, + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + if tc.CommandFn == nil { + return nil, nil + } + return tc.CommandFn(t, s.Tv(), task.History), nil + } + + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + if tc.MessageFn == nil { + return nil, nil + } + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] + return tc.MessageFn(t, s.Tv(), updRequestMsg), nil + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: t, + } + + halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) + defer cancel() + updateResultCh := sendUpdate(halfSecondTimeoutCtx, s, s.Tv()) + + // Process update in workflow. + _, err := poller.PollAndProcessWorkflowTask() + updateResult := <-updateResultCh + if tc.RespondWorkflowTaskError != "" { + s.Error(err, "RespondWorkflowTaskCompleted should return an error contains `%v`", tc.RespondWorkflowTaskError) + s.Contains(err.Error(), tc.RespondWorkflowTaskError) + + var wfNotReady *serviceerror.WorkflowNotReady + s.ErrorAs(updateResult.err, &wfNotReady, "API caller should get serviceerror.WorkflowNotReady, if server got a validation error while processing worker response.") + s.Contains(updateResult.err.Error(), "Unable to perform workflow execution update due to unexpected workflow task failure.") + s.Nil(updateResult.response) + } else { + s.NoError(err) + s.NoError(updateResult.err) + } + }) + } + }) + + t.Run("StickySpeculativeWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } + + // Drain existing first WT from regular task queue, but respond with sticky queue enabled response, next WT will go to sticky queue. + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + StickyAttributes: s.Tv().StickyExecutionAttributes(3 * time.Second), + }, nil + }) + s.NoError(err) + + go func() { + // Process update in workflow task (it is sticky). + res, err := s.TaskPoller(). + PollWorkflowTask(&workflowservice.PollWorkflowTaskQueueRequest{TaskQueue: s.Tv().StickyTaskQueue()}). + HandleTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + // This WT contains partial history because sticky was enabled. + s.EqualHistory(` 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted`, task.History) - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + //nolint:testifylint // callback runs synchronously within HandleTask + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) //nolint:testifylint // callback runs synchronously within HandleTask + s.EqualValues(5, updRequestMsg.GetEventId()) //nolint:testifylint // callback runs synchronously within HandleTask - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, updRequestMsg), - }, nil - }) - s.NoError(err) - s.NotNil(res) - s.EqualValues(0, res.ResetHistoryEventId) - }() + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), + }, nil + }) + //nolint:testifylint // intentional async polling pattern + s.NoError(err) + s.NotNil(res) //nolint:testifylint // intentional async polling pattern + s.EqualValues(0, res.ResetHistoryEventId) //nolint:testifylint // intentional async polling pattern + }() - // This is to make sure that sticky poller above reached server first. - // And when update comes, stick poller is already available. - time.Sleep(500 * time.Millisecond) //nolint:forbidigo - updateResult := <-sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) + // This is to make sure that sticky poller above reached server first. + // And when update comes, stick poller is already available. + time.Sleep(500 * time.Millisecond) //nolint:forbidigo + updateResult := <-sendUpdateNoError(s, tv) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1364,26 +1385,25 @@ func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete() 7 WorkflowTaskCompleted 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} - `, s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution())) - }) - } -} + `, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + }) + } + }) -func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_StickyWorkerUnavailable() { - tv := testvars.New(s.T()) + t.Run("StickySpeculativeWorkflowTask_AcceptComplete_StickyWorkerUnavailable", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - // Worker gets full history because update was issued after sticky worker is gone. - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + // Worker gets full history because update was issued after sticky worker is gone. + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1391,74 +1411,75 @@ func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_S 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - StickyTaskQueue: tv.StickyTaskQueue(), - StickyScheduleToStartTimeout: 3 * time.Second, - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + StickyTaskQueue: s.Tv().StickyTaskQueue(), + StickyScheduleToStartTimeout: 3 * time.Second, + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain existing WT from regular task queue, but respond with sticky enabled response to enable stick task queue. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithRespondSticky, testcore.WithoutRetries) - s.NoError(err) + // Drain existing WT from regular task queue, but respond with sticky enabled response to enable stick task queue. + _, err := poller.PollAndProcessWorkflowTask(testcore.WithRespondSticky, testcore.WithoutRetries) + s.NoError(err) - s.Logger.Info("Sleep 10+ seconds to make sure stickyPollerUnavailableWindow time has passed.") - time.Sleep(10*time.Second + 100*time.Millisecond) //nolint:forbidigo - s.Logger.Info("Sleep 10+ seconds is done.") + s.Logger.Info("Sleep 10+ seconds to make sure stickyPollerUnavailableWindow time has passed.") + time.Sleep(10*time.Second + 100*time.Millisecond) //nolint:forbidigo + s.Logger.Info("Sleep 10+ seconds is done.") - // Now send an update. It should try sticky task queue first, but got "StickyWorkerUnavailable" error - // and resend it to normal. - // This can be observed in wtHandler: if history is partial => sticky task queue is used. - updateResultCh := sendUpdateNoError(s, tv) + // Now send an update. It should try sticky task queue first, but got "StickyWorkerUnavailable" error + // and resend it to normal. + // This can be observed in wtHandler: if history is partial => sticky task queue is used. + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Process update in workflow task from non-sticky task queue. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Process update in workflow task from non-sticky task queue. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1469,96 +1490,95 @@ func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_S 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} `, events) -} + }) -func (s *UpdateWorkflowSuite) TestFirstNormalScheduledWorkflowTask_Reject() { - tv := testvars.New(s.T()) + t.Run("FirstNormalScheduledWorkflowTask_Reject", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted`, task.History) - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(2, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(2, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, updateResp.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, updateResp.ResetHistoryEventId) - s.Equal(1, wtHandlerCalls) - s.Equal(1, msgHandlerCalls) + s.Equal(1, wtHandlerCalls) + s.Equal(1, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled // First normal WT was scheduled before update and therefore all 3 events have to be written even if update was rejected. 3 WorkflowTaskStarted 4 WorkflowTaskCompleted // Empty completed WT. No new events were created after it. `, events) -} + }) -func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_Reject() { - tv := testvars.New(s.T()) + t.Run("EmptySpeculativeWorkflowTask_Reject", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + updateResultCh := sendUpdateNoError(s, s.Tv()) - updateResultCh := sendUpdateNoError(s, tv) - - // Process update in workflow. - res, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Process update in workflow. + res, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1567,30 +1587,30 @@ func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_Reject() { 6 WorkflowTaskStarted `, task.History) - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(5, updRequestMsg.GetEventId()) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, updRequestMsg), - }, nil - }) - s.NoError(err) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(3, res.ResetHistoryEventId) + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), updRequestMsg), + }, nil + }) + s.NoError(err) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(3, res.ResetHistoryEventId) - // Send signal to create WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) + // Send signal to create WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - // Process signal and complete workflow. - res, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Process signal and complete workflow. + res, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1599,22 +1619,22 @@ func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_Reject() { 6 WorkflowTaskScheduled 7 WorkflowTaskStarted`, task.History) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, }, }, - }, - }, nil - }) - s.NoError(err) - s.NotNil(res) + }, nil + }) + s.NoError(err) + s.NotNil(res) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1625,29 +1645,28 @@ func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_Reject() { 8 WorkflowTaskCompleted 9 WorkflowExecutionCompleted `, events) -} + }) -func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_Reject() { - tv := testvars.New(s.T()) + t.Run("NotEmptySpeculativeWorkflowTask_Reject", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv.ActivityID(), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: tv.Any().InfiniteTimeout(), - }}, - }}, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: s.Tv().ActivityID(), + ActivityType: s.Tv().ActivityType(), + TaskQueue: s.Tv().TaskQueue(), + ScheduleToCloseTimeout: s.Tv().Any().InfiniteTimeout(), + }}, + }}, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1656,9 +1675,9 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_Reject() { 6 WorkflowTaskScheduled // Speculative WFT will be written to the history because there is ActivityTaskScheduled(5) event. 7 WorkflowTaskStarted `, task.History) - return nil, nil - case 3: - s.EqualHistory(` + return nil, nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1672,82 +1691,81 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_Reject() { 11 WorkflowTaskScheduled 12 WorkflowTaskStarted `, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1, 3: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(6, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(6, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - case 3: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with normal workflow task") + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with normal workflow task") - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete workflow. - res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) + // Complete workflow. + res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1762,35 +1780,34 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_Reject() { 12 WorkflowTaskStarted 13 WorkflowTaskCompleted 14 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() { - tv := testvars.New(s.T()) + t.Run("1stAccept_2ndAccept_2ndComplete_1stComplete", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1).WithActivityIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2).WithActivityIDNumber(2) - mustStartWorkflow(s, tv) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1).WithActivityIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2).WithActivityIDNumber(2) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted`, task.History) - return append(s.UpdateAcceptCommands(tv1), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv1.ActivityID(), - ActivityType: tv1.ActivityType(), - TaskQueue: tv1.TaskQueue(), - ScheduleToCloseTimeout: tv1.Any().InfiniteTimeout(), - }}, - }), nil - case 2: - s.EqualHistory(` + return append(s.UpdateAcceptCommands(tv1), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv1.ActivityID(), + ActivityType: tv1.ActivityType(), + TaskQueue: tv1.TaskQueue(), + ScheduleToCloseTimeout: tv1.Any().InfiniteTimeout(), + }}, + }), nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1799,17 +1816,17 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() 6 ActivityTaskScheduled 7 WorkflowTaskScheduled // New normal WT is created because of the 2nd update. 8 WorkflowTaskStarted`, task.History) - return append(s.UpdateAcceptCommands(tv2), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv2.ActivityID(), - ActivityType: tv2.ActivityType(), - TaskQueue: tv2.TaskQueue(), - ScheduleToCloseTimeout: tv2.Any().InfiniteTimeout(), - }}, - }), nil - case 3: - s.EqualHistory(` + return append(s.UpdateAcceptCommands(tv2), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv2.ActivityID(), + ActivityType: tv2.ActivityType(), + TaskQueue: tv2.TaskQueue(), + ScheduleToCloseTimeout: tv2.Any().InfiniteTimeout(), + }}, + }), nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1826,9 +1843,9 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() 14 WorkflowTaskScheduled 15 WorkflowTaskStarted `, task.History) - return s.UpdateCompleteCommands(tv2), nil - case 4: - s.EqualHistory(` + return s.UpdateCompleteCommands(tv2), nil + case 4: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1851,103 +1868,104 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() 20 WorkflowTaskScheduled 21 WorkflowTaskStarted `, task.History) - return s.UpdateCompleteCommands(tv1), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateCompleteCommands(tv1), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - var upd1RequestMsg, upd2RequestMsg *protocolpb.Message + var upd1RequestMsg, upd2RequestMsg *protocolpb.Message - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - upd1RequestMsg = task.Messages[0] - upd1Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd1RequestMsg.GetBody()) - s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), upd1Request.GetInput().GetArgs())) - s.EqualValues(2, upd1RequestMsg.GetEventId()) - return s.UpdateAcceptMessages(tv1, upd1RequestMsg), nil - case 2: - upd2RequestMsg = task.Messages[0] - upd2Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd2RequestMsg.GetBody()) - s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), upd2Request.GetInput().GetArgs())) - s.EqualValues(7, upd2RequestMsg.GetEventId()) - return s.UpdateAcceptMessages(tv2, upd2RequestMsg), nil - case 3: - s.NotNil(upd2RequestMsg) - return s.UpdateCompleteMessages(tv2, upd2RequestMsg), nil - case 4: - s.NotNil(upd1RequestMsg) - return s.UpdateCompleteMessages(tv1, upd1RequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + upd1RequestMsg = task.Messages[0] + upd1Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd1RequestMsg.GetBody()) + s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), upd1Request.GetInput().GetArgs())) + s.EqualValues(2, upd1RequestMsg.GetEventId()) + return s.UpdateAcceptMessages(tv1, upd1RequestMsg), nil + case 2: + upd2RequestMsg = task.Messages[0] + upd2Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd2RequestMsg.GetBody()) + s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), upd2Request.GetInput().GetArgs())) + s.EqualValues(7, upd2RequestMsg.GetEventId()) + return s.UpdateAcceptMessages(tv2, upd2RequestMsg), nil + case 3: + s.NotNil(upd2RequestMsg) + return s.UpdateCompleteMessages(tv2, upd2RequestMsg), nil + case 4: + s.NotNil(upd1RequestMsg) + return s.UpdateCompleteMessages(tv1, upd1RequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } - updateResultCh1 := sendUpdateNoError(s, tv1) + updateResultCh1 := sendUpdateNoError(s, tv1) - // Accept update1 in normal WT1. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Accept update1 in normal WT1. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - // Send 2nd update and create speculative WT2. - updateResultCh2 := sendUpdateNoError(s, tv2) + // Send 2nd update and create speculative WT2. + updateResultCh2 := sendUpdateNoError(s, tv2) - // Poll for WT2 which 2nd update. Accept update2. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Poll for WT2 which 2nd update. Accept update2. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete update2 in WT3. - res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult2 := <-updateResultCh2 - s.EqualValues("success-result-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Complete update2 in WT3. + res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult2 := <-updateResultCh2 + s.Equal("success-result-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete update1 in WT4. - res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult1 := <-updateResultCh1 - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult1.Stage) - s.EqualValues("success-result-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updateResult1.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Complete update1 in WT4. + res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult1 := <-updateResultCh1 + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult1.Stage) + s.Equal("success-result-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updateResult1.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(4, wtHandlerCalls) - s.Equal(4, msgHandlerCalls) + s.Equal(4, wtHandlerCalls) + s.Equal(4, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1972,36 +1990,35 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() 22 WorkflowTaskCompleted 23 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 5} // 1st update is completed. `, events) -} + }) -func (s *UpdateWorkflowSuite) Test1stAccept_2ndReject_1stComplete() { - tv := testvars.New(s.T()) + t.Run("1stAccept_2ndReject_1stComplete", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1).WithActivityIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2).WithActivityIDNumber(2) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1).WithActivityIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2).WithActivityIDNumber(2) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted`, task.History) - return append(s.UpdateAcceptCommands(tv1), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv1.ActivityID(), - ActivityType: tv1.ActivityType(), - TaskQueue: tv1.TaskQueue(), - ScheduleToCloseTimeout: tv1.Any().InfiniteTimeout(), - }}, - }), nil - case 2: - s.EqualHistory(` + return append(s.UpdateAcceptCommands(tv1), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv1.ActivityID(), + ActivityType: tv1.ActivityType(), + TaskQueue: tv1.TaskQueue(), + ScheduleToCloseTimeout: tv1.Any().InfiniteTimeout(), + }}, + }), nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2011,10 +2028,10 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndReject_1stComplete() { 7 WorkflowTaskScheduled // Speculative WFT with WorkflowExecutionUpdateAccepted(5) event. 8 WorkflowTaskStarted `, task.History) - // Message handler rejects 2nd update. - return nil, nil - case 3: - s.EqualHistory(` + // Message handler rejects 2nd update. + return nil, nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2029,90 +2046,91 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndReject_1stComplete() { 12 WorkflowTaskScheduled 13 WorkflowTaskStarted `, task.History) - return s.UpdateCompleteCommands(tv1), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateCompleteCommands(tv1), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - var upd1RequestMsg *protocolpb.Message - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - upd1RequestMsg = task.Messages[0] - upd1Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd1RequestMsg.GetBody()) - s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), upd1Request.GetInput().GetArgs())) - s.EqualValues(2, upd1RequestMsg.GetEventId()) - return s.UpdateAcceptMessages(tv1, upd1RequestMsg), nil - case 2: - upd2RequestMsg := task.Messages[0] - upd2Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd2RequestMsg.GetBody()) - s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), upd2Request.GetInput().GetArgs())) - s.EqualValues(7, upd2RequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv2, upd2RequestMsg), nil - case 3: - s.NotNil(upd1RequestMsg) - return s.UpdateCompleteMessages(tv1, upd1RequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + var upd1RequestMsg *protocolpb.Message + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + upd1RequestMsg = task.Messages[0] + upd1Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd1RequestMsg.GetBody()) + s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), upd1Request.GetInput().GetArgs())) + s.EqualValues(2, upd1RequestMsg.GetEventId()) + return s.UpdateAcceptMessages(tv1, upd1RequestMsg), nil + case 2: + upd2RequestMsg := task.Messages[0] + upd2Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd2RequestMsg.GetBody()) + s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), upd2Request.GetInput().GetArgs())) + s.EqualValues(7, upd2RequestMsg.GetEventId()) + return s.UpdateRejectMessages(tv2, upd2RequestMsg), nil + case 3: + s.NotNil(upd1RequestMsg) + return s.UpdateCompleteMessages(tv1, upd1RequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } - updateResultCh1 := sendUpdateNoError(s, tv1) + updateResultCh1 := sendUpdateNoError(s, tv1) - // Accept update1 in WT1. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Accept update1 in WT1. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - // Send 2nd update and create speculative WT2. - updateResultCh2 := sendUpdateNoError(s, tv2) + // Send 2nd update and create speculative WT2. + updateResultCh2 := sendUpdateNoError(s, tv2) - // Poll for WT2 which 2nd update. Reject update2. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with workflow task which had events") + // Poll for WT2 which 2nd update. Reject update2. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with workflow task which had events") - updateResult2 := <-updateResultCh2 - s.Equal("rejection-of-"+tv2.UpdateID(), updateResult2.GetOutcome().GetFailure().GetMessage()) + updateResult2 := <-updateResultCh2 + s.Equal("rejection-of-"+tv2.UpdateID(), updateResult2.GetOutcome().GetFailure().GetMessage()) - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete update1 in WT3. - res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult1 := <-updateResultCh1 - s.EqualValues("success-result-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updateResult1.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Complete update1 in WT3. + res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult1 := <-updateResultCh1 + s.Equal("success-result-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updateResult1.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2129,24 +2147,23 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndReject_1stComplete() { 14 WorkflowTaskCompleted 15 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 5} `, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_Fail", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + timeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 2*time.Second) + defer cancel() + updateResultCh := sendUpdate(timeoutCtx, s, s.Tv()) - timeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 2*time.Second) - defer cancel() - updateResultCh := sendUpdate(timeoutCtx, s, tv) - - // Try to accept update in workflow: get malformed response. - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Try to accept update in workflow: get malformed response. + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2155,36 +2172,37 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 6 WorkflowTaskStarted `, task.History) - updRequestMsg := task.Messages[0] - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: s.UpdateAcceptCommands(tv), - // Emulate bug in worker/SDK update handler code. Return malformed acceptance response. - Messages: []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: tv.Any().String(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: updRequestMsg.GetId(), - AcceptedRequestSequencingEventId: updRequestMsg.GetEventId(), - AcceptedRequest: nil, // must not be nil! - }), + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: s.UpdateAcceptCommands(s.Tv()), + // Emulate bug in worker/SDK update handler code. Return malformed acceptance response. + Messages: []*protocolpb.Message{ + { + Id: s.Tv().MessageID() + "_update-accepted", + ProtocolInstanceId: s.Tv().Any().String(), + SequencingId: nil, + Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ + AcceptedRequestMessageId: updRequestMsg.GetId(), + AcceptedRequestSequencingEventId: updRequestMsg.GetEventId(), + AcceptedRequest: nil, // must not be nil! + }), + }, }, - }, - }, nil - }) - s.Error(err) - s.Contains(err.Error(), "wasn't found") + }, nil + }) + s.Error(err) + s.Contains(err.Error(), "wasn't found") - // Update is aborted, speculative WFT failure is recorded into the history. - updateResult := <-updateResultCh - var wfNotReady *serviceerror.WorkflowNotReady - s.ErrorAs(updateResult.err, &wfNotReady) - s.Contains(updateResult.err.Error(), "Unable to perform workflow execution update due to unexpected workflow task failure.") + // Update is aborted, speculative WFT failure is recorded into the history. + updateResult := <-updateResultCh + var wfNotReady *serviceerror.WorkflowNotReady + s.ErrorAs(updateResult.err, &wfNotReady) + s.Contains(updateResult.err.Error(), "Unable to perform workflow execution update due to unexpected workflow task failure.") - // New transient WFT is created, but it is not shown in the history. - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + // New transient WFT is created, but it is not shown in the history. + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2193,13 +2211,13 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 6 WorkflowTaskStarted 7 WorkflowTaskFailed`, events) - // Send Update again. It will be delivered on existing transient WFT. - updateResultCh = sendUpdate(timeoutCtx, s, tv) + // Send Update again. It will be delivered on existing transient WFT. + updateResultCh = sendUpdate(timeoutCtx, s, s.Tv()) - // Try to accept 2nd update in workflow: get error. Poller will fail WFT, but the registry won't be cleared and Update won't be aborted. - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Try to accept 2nd update in workflow: get error. Poller will fail WFT, but the registry won't be cleared and Update won't be aborted. + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2210,23 +2228,24 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 8 WorkflowTaskScheduled // Transient WFT 9 WorkflowTaskStarted`, task.History) - updRequestMsg := task.Messages[0] - s.EqualValues(8, updRequestMsg.GetEventId()) - // Returning error will cause the poller to fail WFT. - return nil, errors.New("malformed request") - }) - // The error is from RespondWorkflowTaskFailed, which should go w/o error. - s.NoError(err) + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] + s.EqualValues(8, updRequestMsg.GetEventId()) + // Returning error will cause the poller to fail WFT. + return nil, errors.New("malformed request") + }) + // The error is from RespondWorkflowTaskFailed, which should go w/o error. + s.NoError(err) - // Update timed out, but stays in the registry and will be delivered again on the new transient WFT. - updateResult = <-updateResultCh - s.Error(updateResult.err) - s.True(common.IsContextDeadlineExceededErr(updateResult.err), "UpdateWorkflowExecution must timeout after 2 seconds") - s.Nil(updateResult.response) + // Update timed out, but stays in the registry and will be delivered again on the new transient WFT. + updateResult = <-updateResultCh + s.Error(updateResult.err) + s.True(common.IsContextDeadlineExceededErr(updateResult.err), "UpdateWorkflowExecution must timeout after 2 seconds") + s.Nil(updateResult.response) - // This WFT failure wasn't recorded because WFT was transient. - events = s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + // This WFT failure wasn't recorded because WFT was transient. + events = s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2235,22 +2254,23 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 6 WorkflowTaskStarted 7 WorkflowTaskFailed`, events) - // Try to accept 2nd update in workflow 2nd time: get error. Poller will fail WT. Update is not aborted. - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - // 1st attempt UpdateWorkflowExecution call has timed out but the - // update is still running - updRequestMsg := task.Messages[0] - s.EqualValues(8, updRequestMsg.GetEventId()) - // Fail WT one more time. This is transient WT and shouldn't appear in the history. - // Returning error will cause the poller to fail WT. - return nil, errors.New("malformed request") - }) - // The error is from RespondWorkflowTaskFailed, which should go w/o error. - s.NoError(err) + // Try to accept 2nd update in workflow 2nd time: get error. Poller will fail WT. Update is not aborted. + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + // 1st attempt UpdateWorkflowExecution call has timed out but the + // update is still running + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] + s.EqualValues(8, updRequestMsg.GetEventId()) + // Fail WT one more time. This is transient WT and shouldn't appear in the history. + // Returning error will cause the poller to fail WT. + return nil, errors.New("malformed request") + }) + // The error is from RespondWorkflowTaskFailed, which should go w/o error. + s.NoError(err) - events = s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events = s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2259,10 +2279,10 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 6 WorkflowTaskStarted 7 WorkflowTaskFailed`, events) - // Complete Update and workflow. - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Complete Update and workflow. + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2273,18 +2293,19 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 8 WorkflowTaskScheduled // Transient WFT 9 WorkflowTaskStarted`, task.History) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - Commands: append(s.UpdateAcceptCompleteCommands(tv), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }), - }, nil - }) - s.NoError(err) + s.Require().NotEmpty(task.Messages, "expected update message in task") + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + Commands: append(s.UpdateAcceptCompleteCommands(s.Tv()), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }), + }, nil + }) + s.NoError(err) - events = s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events = s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2298,22 +2319,21 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 11 WorkflowExecutionUpdateAccepted 12 WorkflowExecutionUpdateCompleted 13 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_ConvertToNormalBecauseOfBufferedSignal() { - tv := testvars.New(s.T()) + t.Run("StartedSpeculativeWorkflowTask_ConvertToNormalBecauseOfBufferedSignal", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2321,85 +2341,84 @@ func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_ConvertToNormal 5 WorkflowTaskScheduled // Speculative WT. Events 5 and 6 are written into the history when signal is received. 6 WorkflowTaskStarted `, task.History) - // Send signal which will be buffered. This will persist MS and speculative WT must be converted to normal. - err := s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - return nil, nil - case 3: - s.EqualHistory(` + // Send signal which will be buffered. This will persist MS and speculative WT must be converted to normal. + err := s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) + return nil, nil + case 3: + s.EqualHistory(` 7 WorkflowTaskCompleted 8 WorkflowExecutionSignaled // It was buffered and got to the history after WT is completed. 9 WorkflowTaskScheduled 10 WorkflowTaskStarted`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1, 3: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] - s.EqualValues(5, updRequestMsg.GetEventId()) + s.EqualValues(5, updRequestMsg.GetEventId()) - // Update is rejected but corresponding speculative WT will be in the history anyway, because it was converted to normal due to buffered signal. - return s.UpdateRejectMessages(tv, updRequestMsg), nil - case 3: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + // Update is rejected but corresponding speculative WT will be in the history anyway, because it was converted to normal due to buffered signal. + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, updateResp.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, updateResp.ResetHistoryEventId) - // Complete workflow. - completeWorkflowResp, err := poller.HandlePartialWorkflowTask(updateResp.GetWorkflowTask(), false) - s.NoError(err) - s.NotNil(completeWorkflowResp) - s.Nil(completeWorkflowResp.GetWorkflowTask()) - s.EqualValues(0, completeWorkflowResp.ResetHistoryEventId) + // Complete workflow. + completeWorkflowResp, err := poller.HandlePartialWorkflowTask(updateResp.GetWorkflowTask(), false) + s.NoError(err) + s.NotNil(completeWorkflowResp) + s.Nil(completeWorkflowResp.GetWorkflowTask()) + s.EqualValues(0, completeWorkflowResp.ResetHistoryEventId) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2412,22 +2431,21 @@ func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_ConvertToNormal 10 WorkflowTaskStarted 11 WorkflowTaskCompleted 12 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_ConvertToNormalBecauseOfSignal() { - tv := testvars.New(s.T()) + t.Run("ScheduledSpeculativeWorkflowTask_ConvertToNormalBecauseOfSignal", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2435,69 +2453,71 @@ func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_ConvertToNorm 5 WorkflowTaskScheduled // It was initially speculative WT but was already converted to normal when signal was received. 6 WorkflowExecutionSignaled 7 WorkflowTaskStarted`, task.History) - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] - s.EqualValues(6, updRequestMsg.GetEventId()) + s.EqualValues(6, updRequestMsg.GetEventId()) - // Update is rejected but corresponding speculative WT was already converted to normal, - // and will be in the history anyway. - return s.UpdateRejectMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + // Update is rejected but corresponding speculative WT was already converted to normal, + // and will be in the history anyway. + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Send signal which will NOT be buffered because speculative WT is not started yet (only scheduled). - // This will persist MS and speculative WT must be converted to normal. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) + // Send signal which will NOT be buffered because speculative WT is not started yet (only scheduled). + // This will persist MS and speculative WT must be converted to normal. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2507,35 +2527,35 @@ func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_ConvertToNorm 7 WorkflowTaskStarted 8 WorkflowTaskCompleted // Update was rejected but WT events 5,7,8 are in the history because of signal. `, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_StartToCloseTimeout() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_StartToCloseTimeout", func(t *testing.T) { + // Uses CaptureMetricsHandler which requires a dedicated cluster to avoid metric interference. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() - defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) + request := &workflowservice.StartWorkflowExecutionRequest{ + RequestId: s.Tv().Any().String(), + Namespace: s.Namespace().String(), + WorkflowId: s.Tv().WorkflowID(), + WorkflowType: s.Tv().WorkflowType(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskTimeout: durationpb.New(1 * time.Second), // Important! + } - request := &workflowservice.StartWorkflowExecutionRequest{ - RequestId: tv.Any().String(), - Namespace: s.Namespace().String(), - WorkflowId: tv.WorkflowID(), - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskTimeout: durationpb.New(1 * time.Second), // Important! - } + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), request) + s.NoError(err) - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), request) - s.NoError(err) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2543,13 +2563,13 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_StartToCloseTimeout() 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted `, task.History) - // Emulate slow worker: sleep little more than WT timeout. - time.Sleep(request.WorkflowTaskTimeout.AsDuration() + 100*time.Millisecond) //nolint:forbidigo - // This doesn't matter because WT times out before update is applied. - return s.UpdateAcceptCompleteCommands(tv), nil - case 3: - // Speculative WT timed out and retried as normal WT. - s.EqualHistory(` + // Emulate slow worker: sleep little more than WT timeout. + time.Sleep(request.WorkflowTaskTimeout.AsDuration() + 100*time.Millisecond) //nolint:forbidigo + // This doesn't matter because WT times out before update is applied. + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + case 3: + // Speculative WT timed out and retried as normal WT. + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2559,98 +2579,99 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_StartToCloseTimeout() 7 WorkflowTaskTimedOut 8 WorkflowTaskScheduled {"Attempt":2 } // Transient WT. 9 WorkflowTaskStarted`, task.History) - commands := append(s.UpdateAcceptCompleteCommands(tv), - &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }) - return commands, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + commands := append(s.UpdateAcceptCompleteCommands(s.Tv()), + &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }) + return commands, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1) - updRequestMsg := task.Messages[0] + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Len(task.Messages, 1) + updRequestMsg := task.Messages[0] - // This doesn't matter because WT times out before update is applied. - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - case 3: - // Update is still in registry and was sent again. - updRequestMsg := task.Messages[0] + // This doesn't matter because WT times out before update is applied. + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + case 3: + // Update is still in registry and was sent again. + updRequestMsg := task.Messages[0] - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err = poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, tv) - - // Try to process update in workflow, but it takes more than WT timeout. So, WT times out. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.Error(err) - s.Equal("Workflow task not found.", err.Error()) - - // ensure correct metrics were recorded - snap := capture.Snapshot() - - var speculativeWorkflowTaskTimeoutTasks int - for _, m := range snap[metrics.TaskRequests.Name()] { - if m.Tags[metrics.OperationTagName] == metrics.TaskTypeTimerActiveTaskSpeculativeWorkflowTaskTimeout { - speculativeWorkflowTaskTimeoutTasks += 1 + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), } - } - s.Equal(1, speculativeWorkflowTaskTimeoutTasks, "expected 1 speculative workflow task timeout task to be created") - var speculativeStartToCloseTimeouts int - for _, m := range snap[metrics.StartToCloseTimeoutCounter.Name()] { - if m.Tags[metrics.OperationTagName] == metrics.TaskTypeTimerActiveTaskSpeculativeWorkflowTaskTimeout { - speculativeStartToCloseTimeouts += 1 + // Drain first WT. + _, err = poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Try to process update in workflow, but it takes more than WT timeout. So, WT times out. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.Error(err) + s.Equal("Workflow task not found.", err.Error()) + + // ensure correct metrics were recorded + snap := capture.Snapshot() + + var speculativeWorkflowTaskTimeoutTasks int + for _, m := range snap[metrics.TaskRequests.Name()] { + if m.Tags[metrics.OperationTagName] == metrics.TaskTypeTimerActiveTaskSpeculativeWorkflowTaskTimeout { + speculativeWorkflowTaskTimeoutTasks += 1 + } } - } - s.Equal(1, speculativeStartToCloseTimeouts, "expected 1 timeout of a speculative workflow task timeout task") + s.Equal(1, speculativeWorkflowTaskTimeoutTasks, "expected 1 speculative workflow task timeout task to be created") - // New normal WT was created on server after speculative WT has timed out. - // It will accept and complete update first and workflow itself with the same WT. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, updateResp.ResetHistoryEventId) - s.Nil(updateResp.GetWorkflowTask()) + var speculativeStartToCloseTimeouts int + for _, m := range snap[metrics.StartToCloseTimeoutCounter.Name()] { + if m.Tags[metrics.OperationTagName] == metrics.TaskTypeTimerActiveTaskSpeculativeWorkflowTaskTimeout { + speculativeStartToCloseTimeouts += 1 + } + } + s.Equal(1, speculativeStartToCloseTimeouts, "expected 1 timeout of a speculative workflow task timeout task") - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + // New normal WT was created on server after speculative WT has timed out. + // It will accept and complete update first and workflow itself with the same WT. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, updateResp.ResetHistoryEventId) + s.Nil(updateResp.GetWorkflowTask()) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2662,36 +2683,35 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_StartToCloseTimeout() 9 WorkflowTaskStarted 10 WorkflowTaskCompleted 11 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 8} // WTScheduled event which delivered update to the worker. - 12 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 11} + 12 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 11} 13 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_ScheduleToStartTimeout", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + // Drain first WT and respond with sticky enabled response to enable sticky task queue. + stickyScheduleToStartTimeout := 1 * time.Second + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + StickyAttributes: s.Tv().StickyExecutionAttributes(stickyScheduleToStartTimeout), + }, nil + }) + s.NoError(err) - // Drain first WT and respond with sticky enabled response to enable sticky task queue. - stickyScheduleToStartTimeout := 1 * time.Second - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - StickyAttributes: tv.StickyExecutionAttributes(stickyScheduleToStartTimeout), - }, nil - }) - s.NoError(err) + sendUpdateNoError(s, s.Tv()) - sendUpdateNoError(s, tv) + s.Logger.Info("Wait for sticky timeout to fire. Sleep poller.StickyScheduleToStartTimeout+ seconds.", tag.NewDurationTag("StickyScheduleToStartTimeout", stickyScheduleToStartTimeout)) + time.Sleep(stickyScheduleToStartTimeout + 100*time.Millisecond) //nolint:forbidigo + s.Logger.Info("Sleep is done.") - s.Logger.Info("Wait for sticky timeout to fire. Sleep poller.StickyScheduleToStartTimeout+ seconds.", tag.NewDurationTag("StickyScheduleToStartTimeout", stickyScheduleToStartTimeout)) - time.Sleep(stickyScheduleToStartTimeout + 100*time.Millisecond) //nolint:forbidigo - s.Logger.Info("Sleep is done.") - - // Try to process update in workflow, poll from normal task queue. - res, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - // Speculative WFT timed out on sticky task queue. Server sent full history with sticky timeout event. - s.EqualHistory(` + // Try to process update in workflow, poll from normal task queue. + res, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + // Speculative WFT timed out on sticky task queue. Server sent full history with sticky timeout event. + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2701,16 +2721,16 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout 7 WorkflowTaskScheduled {"Attempt":1} // Normal WT. 8 WorkflowTaskStarted`, task.History) - // Reject update, but WFT will still be in the history due to timeout on sticky queue. - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - s.NotNil(res) + // Reject update, but WFT will still be in the history due to timeout on sticky queue. + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + s.NotNil(res) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2721,22 +2741,21 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout 8 WorkflowTaskStarted 9 WorkflowTaskCompleted // Normal WT is completed and events are in the history even update was rejected. `, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeoutOnNormalTaskQueue() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_ScheduleToStartTimeoutOnNormalTaskQueue", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2746,73 +2765,74 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout 7 WorkflowTaskScheduled {"Attempt":1} // Normal WT is scheduled. 8 WorkflowTaskStarted `, task.History) - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(7, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(7, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain existing WT from normal task queue. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) + // Drain existing WT from normal task queue. + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) - // Now send an update. It will create a speculative WT on normal task queue, - // which will time out in 5 seconds and create new normal WT. - updateResultCh := sendUpdateNoError(s, tv) + // Now send an update. It will create a speculative WT on normal task queue, + // which will time out in 5 seconds and create new normal WT. + updateResultCh := sendUpdateNoError(s, s.Tv()) - // TODO: it would be nice to shutdown matching before sending an update to emulate case which is actually being tested here. - // But test infrastructure doesn't support it. 5 seconds sleep will cause same observable effect. - s.Logger.Info("Sleep 5+ seconds to make sure tasks.SpeculativeWorkflowTaskScheduleToStartTimeout time has passed.") - time.Sleep(5*time.Second + 100*time.Millisecond) //nolint:forbidigo - s.Logger.Info("Sleep 5+ seconds is done.") + // TODO: it would be nice to shutdown matching before sending an update to emulate case which is actually being tested here. + // But test infrastructure doesn't support it. 5 seconds sleep will cause same observable effect. + s.Logger.Info("Sleep 5+ seconds to make sure tasks.SpeculativeWorkflowTaskScheduleToStartTimeout time has passed.") + time.Sleep(5*time.Second + 100*time.Millisecond) //nolint:forbidigo + s.Logger.Info("Sleep 5+ seconds is done.") - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with normal workflow task") + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with normal workflow task") - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2823,96 +2843,96 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout 8 WorkflowTaskStarted 9 WorkflowTaskCompleted `, events) -} + }) -func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_TerminateWorkflow() { - tv := testvars.New(s.T()) + t.Run("StartedSpeculativeWorkflowTask_TerminateWorkflow", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + // Terminate workflow while speculative WT is running. + _, err := s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) + s.NoError(err) - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - // Terminate workflow while speculative WT is running. - _, err := s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) - - s.EqualHistory(` + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - oneSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 1*time.Second) - defer cancel() - updateResultCh := sendUpdate(oneSecondTimeoutCtx, s, tv) + oneSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 1*time.Second) + defer cancel() + updateResultCh := sendUpdate(oneSecondTimeoutCtx, s, s.Tv()) - // Process update in workflow. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.Error(err) - s.IsType(err, (*serviceerror.NotFound)(nil)) - s.ErrorContains(err, "Workflow task not found.") + // Process update in workflow. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.Error(err) + s.ErrorAs(err, new(*serviceerror.NotFound)) + s.ErrorContains(err, "Workflow task not found.") - updateResult := <-updateResultCh - s.Error(updateResult.err) - var notFound *serviceerror.NotFound - s.ErrorAs(updateResult.err, ¬Found) - s.ErrorContains(updateResult.err, update.AbortedByWorkflowClosingErr.Error()) - s.Nil(updateResult.response) + updateResult := <-updateResultCh + s.Error(updateResult.err) + var notFound *serviceerror.NotFound + s.ErrorAs(updateResult.err, ¬Found) + s.ErrorContains(updateResult.err, update.AbortedByWorkflowClosingErr.Error()) + s.Nil(updateResult.response) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2922,85 +2942,85 @@ func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_TerminateWorkfl 7 WorkflowTaskFailed 8 WorkflowExecutionTerminated`, events) - msResp, err := s.AdminClient().DescribeMutableState(testcore.NewContext(), &adminservice.DescribeMutableStateRequest{ - Namespace: s.Namespace().String(), - Execution: tv.WorkflowExecution(), - Archetype: chasm.WorkflowArchetype, + msResp, err := s.AdminClient().DescribeMutableState(testcore.NewContext(), &adminservice.DescribeMutableStateRequest{ + Namespace: s.Namespace().String(), + Execution: s.Tv().WorkflowExecution(), + Archetype: chasm.WorkflowArchetype, + }) + s.NoError(err) + s.EqualValues(7, msResp.GetDatabaseMutableState().GetExecutionInfo().GetCompletionEventBatchId(), "completion_event_batch_id should point to WTFailed event") }) - s.NoError(err) - s.EqualValues(7, msResp.GetDatabaseMutableState().GetExecutionInfo().GetCompletionEventBatchId(), "completion_event_batch_id should point to WTFailed event") -} -func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_TerminateWorkflow() { - tv := testvars.New(s.T()) + t.Run("ScheduledSpeculativeWorkflowTask_TerminateWorkflow", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - oneSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 1*time.Second) - defer cancel() - updateResultCh := sendUpdate(oneSecondTimeoutCtx, s, tv) + oneSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 1*time.Second) + defer cancel() + updateResultCh := sendUpdate(oneSecondTimeoutCtx, s, s.Tv()) - // Terminate workflow after speculative WT is scheduled but not started. - _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) + // Terminate workflow after speculative WT is scheduled but not started. + _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) + s.NoError(err) - updateResult := <-updateResultCh - s.Error(updateResult.err) - var notFound *serviceerror.NotFound - s.ErrorAs(updateResult.err, ¬Found) - s.ErrorContains(updateResult.err, update.AbortedByWorkflowClosingErr.Error()) - s.Nil(updateResult.response) + updateResult := <-updateResultCh + s.Error(updateResult.err) + var notFound *serviceerror.NotFound + s.ErrorAs(updateResult.err, ¬Found) + s.ErrorContains(updateResult.err, update.AbortedByWorkflowClosingErr.Error()) + s.Nil(updateResult.response) - s.Equal(1, wtHandlerCalls) - s.Equal(1, msgHandlerCalls) + s.Equal(1, wtHandlerCalls) + s.Equal(1, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -3008,137 +3028,860 @@ func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_TerminateWork 5 WorkflowExecutionTerminated // Speculative WTScheduled event is not written to history if WF is terminated. `, events) - msResp, err := s.AdminClient().DescribeMutableState(testcore.NewContext(), &adminservice.DescribeMutableStateRequest{ - Namespace: s.Namespace().String(), - Execution: tv.WorkflowExecution(), - Archetype: chasm.WorkflowArchetype, + msResp, err := s.AdminClient().DescribeMutableState(testcore.NewContext(), &adminservice.DescribeMutableStateRequest{ + Namespace: s.Namespace().String(), + Execution: s.Tv().WorkflowExecution(), + Archetype: chasm.WorkflowArchetype, + }) + s.NoError(err) + s.EqualValues(5, msResp.GetDatabaseMutableState().GetExecutionInfo().GetCompletionEventBatchId(), "completion_event_batch_id should point to WFTerminated event") }) - s.NoError(err) - s.EqualValues(5, msResp.GetDatabaseMutableState().GetExecutionInfo().GetCompletionEventBatchId(), "completion_event_batch_id should point to WFTerminated event") -} -func (s *UpdateWorkflowSuite) TestCompleteWorkflow_AbortUpdates() { - type testCase struct { - name string - description string - updateErr map[string]string // Update error by completionCommand.Name. - updateFailure string - commands func(tv *testvars.TestVars) []*commandpb.Command - messages func(tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message - } - type completionCommand struct { - name string - finalStatus enumspb.WorkflowExecutionStatus - useRunID bool - command func(tv *testvars.TestVars) *commandpb.Command - } - testCases := []testCase{ - { - name: "update admitted", - description: "update in stateAdmitted must get an error", - updateErr: map[string]string{ - "workflow completed": update.AbortedByWorkflowClosingErr.Error(), - "workflow continued as new without runID": "workflow operation can not be applied because workflow is closing", - "workflow continued as new with runID": "workflow operation can not be applied because workflow is closing", - "workflow failed": update.AbortedByWorkflowClosingErr.Error(), + t.Run("CompleteWorkflow_AbortUpdates", func(t *testing.T) { + type testCase struct { + name string + description string + updateErr map[string]string // Update error by completionCommand.Name. + updateFailure string + commands func(s *testcore.FunctionalTestBase, tv *testvars.TestVars) []*commandpb.Command + messages func(s *testcore.FunctionalTestBase, tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message + } + type completionCommand struct { + name string + finalStatus enumspb.WorkflowExecutionStatus + useRunID bool + command func(tv *testvars.TestVars) *commandpb.Command + } + testCases := []testCase{ + { + name: "update admitted", + description: "update in stateAdmitted must get an error", + updateErr: map[string]string{ + "workflow completed": update.AbortedByWorkflowClosingErr.Error(), + "workflow continued as new without runID": "workflow operation can not be applied because workflow is closing", + "workflow continued as new with runID": "workflow operation can not be applied because workflow is closing", + "workflow failed": update.AbortedByWorkflowClosingErr.Error(), + }, + updateFailure: "", + commands: func(s *testcore.FunctionalTestBase, _ *testvars.TestVars) []*commandpb.Command { return nil }, + messages: func(s *testcore.FunctionalTestBase, _ *testvars.TestVars, _ *protocolpb.Message) []*protocolpb.Message { + return nil + }, }, - updateFailure: "", - commands: func(_ *testvars.TestVars) []*commandpb.Command { return nil }, - messages: func(_ *testvars.TestVars, _ *protocolpb.Message) []*protocolpb.Message { return nil }, - }, - { - name: "update accepted", - description: "update in stateAccepted must get an update failure", - updateErr: map[string]string{"*": ""}, - updateFailure: "Workflow Update failed because the Workflow completed before the Update completed.", - commands: func(tv *testvars.TestVars) []*commandpb.Command { return s.UpdateAcceptCommands(tv) }, - messages: func(tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { - return s.UpdateAcceptMessages(tv, updRequestMsg) + { + name: "update accepted", + description: "update in stateAccepted must get an update failure", + updateErr: map[string]string{"*": ""}, + updateFailure: "Workflow Update failed because the Workflow completed before the Update completed.", + commands: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars) []*commandpb.Command { + return s.UpdateAcceptCommands(tv) + }, + messages: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { + return s.UpdateAcceptMessages(tv, updRequestMsg) + }, }, - }, - { - name: "update completed", - description: "completed update must not be affected by workflow completion", - updateErr: map[string]string{"*": ""}, - updateFailure: "", - commands: func(tv *testvars.TestVars) []*commandpb.Command { return s.UpdateAcceptCompleteCommands(tv) }, - messages: func(tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg) + { + name: "update completed", + description: "completed update must not be affected by workflow completion", + updateErr: map[string]string{"*": ""}, + updateFailure: "", + commands: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars) []*commandpb.Command { + return s.UpdateAcceptCompleteCommands(tv) + }, + messages: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { + return s.UpdateAcceptCompleteMessages(tv, updRequestMsg) + }, }, - }, - { - name: "update rejected", - description: "rejected update must be rejected with rejection from workflow", - updateErr: map[string]string{"*": ""}, - updateFailure: "rejection-of-", // Rejection from workflow. - commands: func(tv *testvars.TestVars) []*commandpb.Command { return nil }, - messages: func(tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { - return s.UpdateRejectMessages(tv, updRequestMsg) + { + name: "update rejected", + description: "rejected update must be rejected with rejection from workflow", + updateErr: map[string]string{"*": ""}, + updateFailure: "rejection-of-", // Rejection from workflow. + commands: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars) []*commandpb.Command { return nil }, + messages: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { + return s.UpdateRejectMessages(tv, updRequestMsg) + }, }, - }, - } + } - workflowCompletionCommands := []completionCommand{ - { - name: "workflow completed", - finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - useRunID: false, - command: func(_ *testvars.TestVars) *commandpb.Command { - return &commandpb.Command{ + workflowCompletionCommands := []completionCommand{ + { + name: "workflow completed", + finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + useRunID: false, + command: func(_ *testvars.TestVars) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + } + }, + }, + { + name: "workflow continued as new with runID", + finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_CONTINUED_AS_NEW, + useRunID: true, + command: func(tv *testvars.TestVars) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + }}, + } + }, + }, + { + name: "workflow continued as new without runID", + finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, // This is the status of new run because update doesn't go to particular runID. + useRunID: false, + command: func(tv *testvars.TestVars) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + }}, + } + }, + }, + { + name: "workflow failed", + finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_FAILED, + useRunID: true, + command: func(tv *testvars.TestVars) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_FailWorkflowExecutionCommandAttributes{FailWorkflowExecutionCommandAttributes: &commandpb.FailWorkflowExecutionCommandAttributes{ + Failure: tv.Any().ApplicationFailure(), + }}, + } + }, + }, + } + + for _, tc := range testCases { + for _, wfCC := range workflowCompletionCommands { + t.Run(tc.name+" "+wfCC.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if wfCC.useRunID { + tv = tv.WithRunID(runID) + } + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + return append(tc.commands(s.FunctionalTestBase, s.Tv()), wfCC.command(s.Tv())), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + return tc.messages(s.FunctionalTestBase, s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResultCh := sendUpdate(testcore.NewContext(), s, tv) + + // Complete workflow. + _, err = poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResult := <-updateResultCh + expectedUpdateErr := tc.updateErr[wfCC.name] + if expectedUpdateErr == "" { + expectedUpdateErr = tc.updateErr["*"] + } + if expectedUpdateErr != "" { + s.Error(updateResult.err, tc.description) + s.Equal(expectedUpdateErr, updateResult.err.Error()) + } else { + s.NoError(updateResult.err, tc.description) + } + + if tc.updateFailure != "" { + s.NotNil(updateResult.response.GetOutcome().GetFailure(), tc.description) + s.Contains(updateResult.response.GetOutcome().GetFailure().GetMessage(), tc.updateFailure, tc.description) + } else { + s.Nil(updateResult.response.GetOutcome().GetFailure(), tc.description) + } + + if expectedUpdateErr == "" && tc.updateFailure == "" { + s.Equal(runID, updateResult.response.GetUpdateRef().GetWorkflowExecution().GetRunId(), "update wasn't applied to the same run as was started") + } + + // Check that update didn't block workflow completion. + descResp, err := s.FrontendClient().DescribeWorkflowExecution(testcore.NewContext(), &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + Execution: tv.WorkflowExecution(), + }) + s.NoError(err) + s.Equal(wfCC.finalStatus, descResp.WorkflowExecutionInfo.Status) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + }) + } + } + }) + + t.Run("SpeculativeWorkflowTask_Heartbeat", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + // Drain first WT. + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Heartbeat from speculative WT (no messages, no commands). + var updRequestMsg *protocolpb.Message + res, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Events (5 and 6) are for speculative WT, but they won't disappear after reject because speculative WT is converted to normal during heartbeat. + 6 WorkflowTaskStarted + `, task.History) + + s.Len(task.Messages, 1) + updRequestMsg = task.Messages[0] + s.EqualValues(5, updRequestMsg.GetEventId()) + + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + ReturnNewWorkflowTask: true, + ForceCreateNewWorkflowTask: true, + }, nil + }) + s.NoError(err) + + // Reject update from workflow. + updateResp, err := s.TaskPoller().HandleWorkflowTask(s.Tv(), + res.GetWorkflowTask(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` + 7 WorkflowTaskCompleted + 8 WorkflowTaskScheduled // New WT (after heartbeat) is normal and won't disappear from the history after reject. + 9 WorkflowTaskStarted + `, task.History) + + s.Empty(task.Messages) + + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), updRequestMsg), + }, nil + }) + s.NoError(err) + s.NotNil(updateResp) + + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, updateResp.ResetHistoryEventId, "no reset of event ID should happened after update rejection because of heartbeat") + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled + 6 WorkflowTaskStarted + 7 WorkflowTaskCompleted // Heartbeat response. + 8 WorkflowTaskScheduled + 9 WorkflowTaskStarted + 10 WorkflowTaskCompleted // After heartbeat new normal WT was created and events are written into the history even update is rejected. + `, events) + }) + + t.Run("ScheduledSpeculativeWorkflowTask_LostUpdate", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionSignaled + 6 WorkflowTaskScheduled + 7 WorkflowTaskStarted`, task.History) + return []*commandpb.Command{{ CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - } - }, - }, - { - name: "workflow continued as new with runID", - finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_CONTINUED_AS_NEW, - useRunID: true, - command: func(tv *testvars.TestVars) *commandpb.Command { - return &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), - }}, - } - }, - }, - { - name: "workflow continued as new without runID", - finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, // This is the status of new run because update doesn't go to particular runID. - useRunID: false, - command: func(tv *testvars.TestVars) *commandpb.Command { - return &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), - }}, - } - }, - }, - { - name: "workflow failed", - finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_FAILED, - useRunID: true, - command: func(tv *testvars.TestVars) *commandpb.Command { - return &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_FailWorkflowExecutionCommandAttributes{FailWorkflowExecutionCommandAttributes: &commandpb.FailWorkflowExecutionCommandAttributes{ - Failure: tv.Any().ApplicationFailure(), - }}, - } - }, - }, - } + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } - for _, tc := range testCases { - for _, wfCC := range workflowCompletionCommands { - s.Run(tc.name+" "+wfCC.name, func() { - tv := testvars.New(s.T()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Empty(task.Messages, "update lost due to lost update registry") + return nil, nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } - runID := mustStartWorkflow(s, tv) + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) + defer cancel() + updateResult := <-sendUpdate(halfSecondTimeoutCtx, s, s.Tv()) + s.Error(updateResult.err) + s.True(common.IsContextDeadlineExceededErr(updateResult.err), updateResult.err.Error()) + s.Nil(updateResult.response) + + // Lose update registry. Speculative WFT and update registry disappear. + loseUpdateRegistryAndAbandonPendingUpdates(s, s.Tv()) + + // Ensure, there is no WFT. + pollCtx, cancel := context.WithTimeout(testcore.NewContext(), common.MinLongPollTimeout*2) + defer cancel() + pollResponse, err := s.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + }) + s.NoError(err) + s.Nil(pollResponse.Messages, "there should not be new WFT with messages") + + // Send signal to schedule new WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) + + // Complete workflow and check that there is update messages. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + s.NotNil(completeWorkflowResp) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionSignaled + 6 WorkflowTaskScheduled + 7 WorkflowTaskStarted + 8 WorkflowTaskCompleted + 9 WorkflowExecutionCompleted`, events) + }) + + t.Run("StartedSpeculativeWorkflowTask_LostUpdate", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Speculative WT. Events 5 and 6 will be lost. + 6 WorkflowTaskStarted +`, task.History) + + // Lose update registry. Update is lost and NotFound error will be returned to RespondWorkflowTaskCompleted. + loseUpdateRegistryAndAbandonPendingUpdates(s, s.Tv()) + + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + case 3: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionSignaled + 6 WorkflowTaskScheduled + 7 WorkflowTaskStarted +`, task.History) + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + s.EqualValues(5, updRequestMsg.GetEventId()) + + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + case 3: + s.Empty(task.Messages, "no messages since update registry was lost") + return nil, nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) + defer cancel() + updateResultCh := sendUpdate(halfSecondTimeoutCtx, s, s.Tv()) + + // Process update in workflow. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.Error(err) + s.ErrorAs(err, new(*serviceerror.NotFound)) + s.ErrorContains(err, "Workflow task not found") + + updateResult := <-updateResultCh + s.Error(updateResult.err) + s.True(common.IsContextDeadlineExceededErr(updateResult.err), updateResult.err.Error()) + s.Nil(updateResult.response) + + // Send signal to schedule new WFT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) + + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + s.NotNil(completeWorkflowResp) + + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionSignaled + 6 WorkflowTaskScheduled + 7 WorkflowTaskStarted + 8 WorkflowTaskCompleted + 9 WorkflowExecutionCompleted`, events) + }) + + t.Run("FirstNormalWorkflowTask_UpdateResurrectedAfterRegistryCleared", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted +`, task.History) + // Clear update registry. Update will be resurrected in registry from acceptance message. + clearUpdateRegistryAndAbortPendingUpdates(s, s.Tv()) + + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + case 2: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionUpdateAccepted + 6 WorkflowExecutionUpdateCompleted + 7 WorkflowExecutionSignaled + 8 WorkflowTaskScheduled + 9 WorkflowTaskStarted +`, task.History) + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + updRequestMsg := task.Messages[0] + s.EqualValues(2, updRequestMsg.GetEventId()) + + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + case 2: + s.Empty(task.Messages, "update must be processed and not delivered again") + return nil, nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Process update in workflow. Update won't be found on server but will be resurrected from acceptance message and completed. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + + // Client receives resurrected Update outcome. + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + + // Signal to create new WFT which shouldn't get any updates. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) + + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(completeWorkflowResp) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionUpdateAccepted + 6 WorkflowExecutionUpdateCompleted + 7 WorkflowExecutionSignaled + 8 WorkflowTaskScheduled + 9 WorkflowTaskStarted + 10 WorkflowTaskCompleted + 11 WorkflowExecutionCompleted`, events) + }) + + t.Run("ScheduledSpeculativeWorkflowTask_DeduplicateID", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Speculative WT. + 6 WorkflowTaskStarted +`, task.History) + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Len(task.Messages, 1, "2nd update must be deduplicated by ID") + updRequestMsg := task.Messages[0] + + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Send second update with the same ID. + updateResultCh2 := sendUpdateNoError(s, s.Tv()) + + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + updateResult2 := <-updateResultCh2 + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) + s.EqualValues(0, updateResp.ResetHistoryEventId) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled + 6 WorkflowTaskStarted + 7 WorkflowTaskCompleted + 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. + 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} +`, events) + }) + + t.Run("StartedSpeculativeWorkflowTask_DeduplicateID", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + var updateResultCh2 <-chan *workflowservice.UpdateWorkflowExecutionResponse + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + // Send second update with the same ID when WT is started but not completed. + updateResultCh2 = sendUpdateNoError(s, s.Tv()) + + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Speculative WT. + 6 WorkflowTaskStarted +`, task.History) + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Len(task.Messages, 1, "2nd update should not has reached server yet") + updRequestMsg := task.Messages[0] + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) + + updateResult2 := <-updateResultCh2 + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled + 6 WorkflowTaskStarted + 7 WorkflowTaskCompleted + 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. + 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} +`, events) + }) + + t.Run("CompletedSpeculativeWorkflowTask_DeduplicateID", func(t *testing.T) { + testCases := []struct { + Name string + CloseShard bool + }{ + { + Name: "no shard reload", + CloseShard: false, + }, + { + Name: "with shard reload", + CloseShard: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + // Uses closeShard conditionally which requires a dedicated cluster. + var opts []testcore.TestOption + if tc.CloseShard { + opts = append(opts, testcore.WithDedicatedCluster()) + } + s := testcore.NewEnv(t, opts...) + mustStartWorkflow(s, s.Tv()) wtHandlerCalls := 0 wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { @@ -3148,7 +3891,20 @@ func (s *UpdateWorkflowSuite) TestCompleteWorkflow_AbortUpdates() { // Completes first WT with empty command list. return nil, nil case 2: - return append(tc.commands(tv), wfCC.command(tv)), nil + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Speculative WT. + 6 WorkflowTaskStarted +`, task.History) + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + case 3: + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil default: s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) return nil, nil @@ -3163,18 +3919,22 @@ func (s *UpdateWorkflowSuite) TestCompleteWorkflow_AbortUpdates() { return nil, nil case 2: updRequestMsg := task.Messages[0] - return tc.messages(tv, updRequestMsg), nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + case 3: + s.Empty(task.Messages, "2nd update must be deduplicated by ID ") + return nil, nil default: s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) return nil, nil } } + //nolint:staticcheck // SA1019 TaskPoller replacement needed poller := &testcore.TaskPoller{ Client: s.FrontendClient(), Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), WorkflowTaskHandler: wtHandler, MessageHandler: msgHandler, Logger: s.Logger, @@ -3185,779 +3945,53 @@ func (s *UpdateWorkflowSuite) TestCompleteWorkflow_AbortUpdates() { _, err := poller.PollAndProcessWorkflowTask() s.NoError(err) - updateResultCh := sendUpdate(testcore.NewContext(), s, s.useRunID(tv, wfCC.useRunID, runID)) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Complete workflow. + // Process update in workflow. _, err = poller.PollAndProcessWorkflowTask() s.NoError(err) - updateResult := <-updateResultCh - expectedUpdateErr := tc.updateErr[wfCC.name] - if expectedUpdateErr == "" { - expectedUpdateErr = tc.updateErr["*"] - } - if expectedUpdateErr != "" { - s.Error(updateResult.err, tc.description) - s.Equal(updateResult.err.Error(), expectedUpdateErr) - } else { - s.NoError(updateResult.err, tc.description) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + + if tc.CloseShard { + // Close shard to make sure that for completed updates deduplication works even after shard reload. + closeShard(s, s.Tv().WorkflowID()) } - if tc.updateFailure != "" { - s.NotNil(updateResult.response.GetOutcome().GetFailure(), tc.description) - s.Contains(updateResult.response.GetOutcome().GetFailure().GetMessage(), tc.updateFailure, tc.description) - } else { - s.Nil(updateResult.response.GetOutcome().GetFailure(), tc.description) - } + // Send second update with the same ID. It must return immediately. + updateResult2 := <-sendUpdateNoError(s, s.Tv()) - if expectedUpdateErr == "" && tc.updateFailure == "" { - s.Equal(runID, updateResult.response.GetUpdateRef().GetWorkflowExecution().GetRunId(), "update wasn't applied to the same run as was started") - } - - // Check that update didn't block workflow completion. - descResp, err := s.FrontendClient().DescribeWorkflowExecution(testcore.NewContext(), &workflowservice.DescribeWorkflowExecutionRequest{ + // Ensure, there is no new WT. + pollCtx, cancel := context.WithTimeout(testcore.NewContext(), common.MinLongPollTimeout*2) + defer cancel() + pollResponse, err := s.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ Namespace: s.Namespace().String(), - Execution: s.useRunID(tv, wfCC.useRunID, runID).WorkflowExecution(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), }) s.NoError(err) - s.Equal(wfCC.finalStatus, descResp.WorkflowExecutionInfo.Status) + s.Nil(pollResponse.Messages, "there must be no new WT") - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - }) - } - } -} + s.Equal( + "success-result-of-"+s.Tv().UpdateID(), + testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess()), + "results of the first update must be available") -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Heartbeat() { - tv := testvars.New(s.T()) + // Send signal to schedule new WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - mustStartWorkflow(s, tv) + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + s.NotNil(completeWorkflowResp) - // Drain first WT. - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - updateResultCh := sendUpdateNoError(s, tv) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - // Heartbeat from speculative WT (no messages, no commands). - var updRequestMsg *protocolpb.Message - res, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Events (5 and 6) are for speculative WT, but they won't disappear after reject because speculative WT is converted to normal during heartbeat. - 6 WorkflowTaskStarted - `, task.History) - - s.Len(task.Messages, 1) - updRequestMsg = task.Messages[0] - s.EqualValues(5, updRequestMsg.GetEventId()) - - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - ReturnNewWorkflowTask: true, - ForceCreateNewWorkflowTask: true, - }, nil - }) - s.NoError(err) - - // Reject update from workflow. - updateResp, err := s.TaskPoller().HandleWorkflowTask(tv, - res.GetWorkflowTask(), - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` - 7 WorkflowTaskCompleted - 8 WorkflowTaskScheduled // New WT (after heartbeat) is normal and won't disappear from the history after reject. - 9 WorkflowTaskStarted - `, task.History) - - s.Empty(task.Messages) - - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, updRequestMsg), - }, nil - }) - s.NoError(err) - s.NotNil(updateResp) - - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, updateResp.ResetHistoryEventId, "no reset of event ID should happened after update rejection because of heartbeat") - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled - 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted // Heartbeat response. - 8 WorkflowTaskScheduled - 9 WorkflowTaskStarted - 10 WorkflowTaskCompleted // After heartbeat new normal WT was created and events are written into the history even update is rejected. - `, events) -} - -func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_LostUpdate() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionSignaled - 6 WorkflowTaskScheduled - 7 WorkflowTaskStarted`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Empty(task.Messages, "update lost due to lost update registry") - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) - defer cancel() - updateResult := <-sendUpdate(halfSecondTimeoutCtx, s, tv) - s.Error(updateResult.err) - s.True(common.IsContextDeadlineExceededErr(updateResult.err), updateResult.err.Error()) - s.Nil(updateResult.response) - - // Lose update registry. Speculative WFT and update registry disappear. - s.loseUpdateRegistryAndAbandonPendingUpdates(tv) - - // Ensure, there is no WFT. - pollCtx, cancel := context.WithTimeout(testcore.NewContext(), common.MinLongPollTimeout*2) - defer cancel() - pollResponse, err := s.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - }) - s.NoError(err) - s.Nil(pollResponse.Messages, "there should not be new WFT with messages") - - // Send signal to schedule new WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - - // Complete workflow and check that there is update messages. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - s.NotNil(completeWorkflowResp) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionSignaled - 6 WorkflowTaskScheduled - 7 WorkflowTaskStarted - 8 WorkflowTaskCompleted - 9 WorkflowExecutionCompleted`, events) -} - -func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_LostUpdate() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Speculative WT. Events 5 and 6 will be lost. - 6 WorkflowTaskStarted -`, task.History) - - // Lose update registry. Update is lost and NotFound error will be returned to RespondWorkflowTaskCompleted. - s.loseUpdateRegistryAndAbandonPendingUpdates(tv) - - return s.UpdateAcceptCompleteCommands(tv), nil - case 3: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionSignaled - 6 WorkflowTaskScheduled - 7 WorkflowTaskStarted -`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - s.EqualValues(5, updRequestMsg.GetEventId()) - - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - case 3: - s.Empty(task.Messages, "no messages since update registry was lost") - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) - defer cancel() - updateResultCh := sendUpdate(halfSecondTimeoutCtx, s, tv) - - // Process update in workflow. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.Error(err) - s.IsType(&serviceerror.NotFound{}, err) - s.ErrorContains(err, "Workflow task not found") - - updateResult := <-updateResultCh - s.Error(updateResult.err) - s.True(common.IsContextDeadlineExceededErr(updateResult.err), updateResult.err.Error()) - s.Nil(updateResult.response) - - // Send signal to schedule new WFT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - s.NotNil(completeWorkflowResp) - - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionSignaled - 6 WorkflowTaskScheduled - 7 WorkflowTaskStarted - 8 WorkflowTaskCompleted - 9 WorkflowExecutionCompleted`, events) -} - -func (s *UpdateWorkflowSuite) TestFirstNormalWorkflowTask_UpdateResurrectedAfterRegistryCleared() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted -`, task.History) - // Clear update registry. Update will be resurrected in registry from acceptance message. - s.clearUpdateRegistryAndAbortPendingUpdates(tv) - - return s.UpdateAcceptCompleteCommands(tv), nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionUpdateCompleted - 7 WorkflowExecutionSignaled - 8 WorkflowTaskScheduled - 9 WorkflowTaskStarted -`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - updRequestMsg := task.Messages[0] - s.EqualValues(2, updRequestMsg.GetEventId()) - - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - case 2: - s.Empty(task.Messages, "update must be processed and not delivered again") - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - updateResultCh := sendUpdateNoError(s, tv) - - // Process update in workflow. Update won't be found on server but will be resurrected from acceptance message and completed. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - - // Client receives resurrected Update outcome. - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - - // Signal to create new WFT which shouldn't get any updates. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(completeWorkflowResp) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionUpdateCompleted - 7 WorkflowExecutionSignaled - 8 WorkflowTaskScheduled - 9 WorkflowTaskStarted - 10 WorkflowTaskCompleted - 11 WorkflowExecutionCompleted`, events) -} - -func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_DeduplicateID() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Speculative WT. - 6 WorkflowTaskStarted -`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1, "2nd update must be deduplicated by ID") - updRequestMsg := task.Messages[0] - - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, tv) - - // Send second update with the same ID. - updateResultCh2 := sendUpdateNoError(s, tv) - - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - updateResult2 := <-updateResultCh2 - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) - s.EqualValues(0, updateResp.ResetHistoryEventId) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled - 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted - 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. - 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} -`, events) -} - -func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_DeduplicateID() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - var updateResultCh2 <-chan *workflowservice.UpdateWorkflowExecutionResponse - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - // Send second update with the same ID when WT is started but not completed. - updateResultCh2 = sendUpdateNoError(s, tv) - - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Speculative WT. - 6 WorkflowTaskStarted -`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1, "2nd update should not has reached server yet") - updRequestMsg := task.Messages[0] - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, tv) - - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) - - updateResult2 := <-updateResultCh2 - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled - 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted - 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. - 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} -`, events) -} - -func (s *UpdateWorkflowSuite) TestCompletedSpeculativeWorkflowTask_DeduplicateID() { - testCases := []struct { - Name string - CloseShard bool - }{ - { - Name: "no shard reload", - CloseShard: false, - }, - { - Name: "with shard reload", - CloseShard: true, - }, - } - - for _, tc := range testCases { - s.Run(tc.Name, func() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Speculative WT. - 6 WorkflowTaskStarted -`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - case 3: - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - case 3: - s.Empty(task.Messages, "2nd update must be deduplicated by ID ") - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, tv) - - // Process update in workflow. - _, err = poller.PollAndProcessWorkflowTask() - s.NoError(err) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - - if tc.CloseShard { - // Close shard to make sure that for completed updates deduplication works even after shard reload. - s.closeShard(tv.WorkflowID()) - } - - // Send second update with the same ID. It must return immediately. - updateResult2 := <-sendUpdateNoError(s, tv) - - // Ensure, there is no new WT. - pollCtx, cancel := context.WithTimeout(testcore.NewContext(), common.MinLongPollTimeout*2) - defer cancel() - pollResponse, err := s.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - }) - s.NoError(err) - s.Nil(pollResponse.Messages, "there must be no new WT") - - s.EqualValues( - "success-result-of-"+tv.UpdateID(), - testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess()), - "results of the first update must be available") - - // Send signal to schedule new WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - s.NotNil(completeWorkflowResp) - - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -3973,79 +4007,80 @@ func (s *UpdateWorkflowSuite) TestCompletedSpeculativeWorkflowTask_DeduplicateID 13 WorkflowTaskCompleted 14 WorkflowExecutionCompleted `, events) - }) - } -} - -func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDifferentStartedId() { - /* - Test scenario: - An update created a speculative WT and WT is dispatched to the worker (started). - Update registry is cleared, speculative WT disappears from server. - Update is retired and second speculative WT is scheduled but not dispatched yet. - An activity completes, it converts the 2nd speculative WT into normal one. - The first speculative WT responds back, server fails request because WorkflowTaskStarted event Id is mismatched. - The second speculative WT responds back and server completes it. - */ - - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Schedule activity. - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv.ActivityID(), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: tv.Any().InfiniteTimeout(), - }}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + }) } - } - - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - // First WT will schedule activity. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - - // Send 1st update. It will create 2nd WT as speculative. - sendUpdateNoError(s, tv) - - // Poll 2nd speculative WT with 1st update. - wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), }) - s.NoError(err) - s.NotNil(wt2) - s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") - s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") - s.EqualValues(7, wt2.StartedEventId) - s.EqualValues(6, wt2.Messages[0].GetEventId()) - s.EqualHistory(` + + t.Run("StaleSpeculativeWorkflowTask_Fail_BecauseOfDifferentStartedId", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + /* + Test scenario: + An update created a speculative WT and WT is dispatched to the worker (started). + Update registry is cleared, speculative WT disappears from server. + Update is retired and second speculative WT is scheduled but not dispatched yet. + An activity completes, it converts the 2nd speculative WT into normal one. + The first speculative WT responds back, server fails request because WorkflowTaskStarted event Id is mismatched. + The second speculative WT responds back and server completes it. + */ + + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Schedule activity. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: s.Tv().ActivityID(), + ActivityType: s.Tv().ActivityType(), + TaskQueue: s.Tv().TaskQueue(), + ScheduleToCloseTimeout: s.Tv().Any().InfiniteTimeout(), + }}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + // First WT will schedule activity. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + + // Send 1st update. It will create 2nd WT as speculative. + sendUpdateNoError(s, s.Tv()) + + // Poll 2nd speculative WT with 1st update. + wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt2) + s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") + s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") + s.EqualValues(7, wt2.StartedEventId) + s.EqualValues(6, wt2.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4054,28 +4089,28 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 6 WorkflowTaskScheduled 7 WorkflowTaskStarted`, wt2.History) - // Clear update registry. Speculative WFT disappears from server. - s.clearUpdateRegistryAndAbortPendingUpdates(tv) + // Clear update registry. Speculative WFT disappears from server. + clearUpdateRegistryAndAbortPendingUpdates(s, s.Tv()) - // Wait for update request to be retry by frontend and recreated in registry. This will create a 3rd WFT as speculative. - waitUpdateAdmitted(s, tv) + // Wait for update request to be retry by frontend and recreated in registry. This will create a 3rd WFT as speculative. + waitUpdateAdmitted(s, s.Tv()) - // Before polling for the 3rd speculative WT, process activity. This will convert 3rd speculative WT to normal WT. - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + // Before polling for the 3rd speculative WT, process activity. This will convert 3rd speculative WT to normal WT. + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Poll the 3rd WFT (not speculative anymore) but must have 2nd update. - wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - }) - s.NoError(err) - s.NotNil(wt3) - s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") - s.Len(wt3.Messages, 1, "3rd workflow task must have a message with 2nd update") - s.EqualValues(9, wt3.StartedEventId) - s.EqualValues(8, wt3.Messages[0].GetEventId()) - s.EqualHistory(` + // Poll the 3rd WFT (not speculative anymore) but must have 2nd update. + wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt3) + s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") + s.Len(wt3.Messages, 1, "3rd workflow task must have a message with 2nd update") + s.EqualValues(9, wt3.StartedEventId) + s.EqualValues(8, wt3.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4086,28 +4121,28 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 8 ActivityTaskCompleted 9 WorkflowTaskStarted`, wt3.History) - // Now try to complete 2nd WT (speculative). It should fail because WorkflowTaskStarted event Id is mismatched. - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt2.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, wt2.Messages[0]), - }) - s.Error(err, "Must fail because WorkflowTaskStarted event Id is different.") - s.IsType(&serviceerror.NotFound{}, err) - s.Contains(err.Error(), "Workflow task not found") + // Now try to complete 2nd WT (speculative). It should fail because WorkflowTaskStarted event Id is mismatched. + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt2.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), wt2.Messages[0]), + }) + s.Error(err, "Must fail because WorkflowTaskStarted event Id is different.") + s.ErrorAs(err, new(*serviceerror.NotFound)) + s.Contains(err.Error(), "Workflow task not found") - // Complete 3rd WT. It should succeed. - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt3.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, wt3.Messages[0]), - }) - s.NoError(err) + // Complete 3rd WT. It should succeed. + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt3.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), wt3.Messages[0]), + }) + s.NoError(err) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4121,61 +4156,62 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 11 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId":8} 12 WorkflowExecutionUpdateCompleted `, events) -} - -func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDifferentStartTime() { - /* - Test scenario: - An update created a speculative WT and WT is dispatched to the worker (started). - WF context is cleared, speculative WT is disappeared from server. - Update is retried and second speculative WT is dispatched to worker with same WT scheduled/started Id and update Id. - The first speculative WT respond back, server reject it because startTime is different. - The second speculative WT respond back, server accept it. - */ - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - Logger: s.Logger, - T: s.T(), - } - - // First WT will schedule activity. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - - // Send update. It will create 2nd WT as speculative. - sendUpdateNoError(s, tv) - - // Poll 2nd speculative WT with 1st update. - wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), }) - s.NoError(err) - s.NotNil(wt2) - s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") - s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") - s.EqualValues(6, wt2.StartedEventId) - s.EqualValues(5, wt2.Messages[0].GetEventId()) - s.EqualHistory(` + + t.Run("StaleSpeculativeWorkflowTask_Fail_BecauseOfDifferentStartTime", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + /* + Test scenario: + An update created a speculative WT and WT is dispatched to the worker (started). + WF context is cleared, speculative WT is disappeared from server. + Update is retried and second speculative WT is dispatched to worker with same WT scheduled/started Id and update Id. + The first speculative WT respond back, server reject it because startTime is different. + The second speculative WT respond back, server accept it. + */ + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + Logger: s.Logger, + T: s.T(), + } + + // First WT will schedule activity. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + + // Send update. It will create 2nd WT as speculative. + sendUpdateNoError(s, s.Tv()) + + // Poll 2nd speculative WT with 1st update. + wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt2) + s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") + s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") + s.EqualValues(6, wt2.StartedEventId) + s.EqualValues(5, wt2.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4183,24 +4219,24 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, wt2.History) - // Clear update registry. Speculative WFT disappears from server. - s.clearUpdateRegistryAndAbortPendingUpdates(tv) + // Clear update registry. Speculative WFT disappears from server. + clearUpdateRegistryAndAbortPendingUpdates(s, s.Tv()) - // Wait for update request to be retry by frontend and recreated in registry. This will create a 3rd WFT as speculative. - waitUpdateAdmitted(s, tv) + // Wait for update request to be retry by frontend and recreated in registry. This will create a 3rd WFT as speculative. + waitUpdateAdmitted(s, s.Tv()) - // Poll for the 3rd speculative WT. - wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - }) - s.NoError(err) - s.NotNil(wt3) - s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") - s.Len(wt3.Messages, 1, "3rd workflow task must have a message with 1st update") - s.EqualValues(6, wt3.StartedEventId) - s.EqualValues(5, wt3.Messages[0].GetEventId()) - s.EqualHistory(` + // Poll for the 3rd speculative WT. + wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt3) + s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") + s.Len(wt3.Messages, 1, "3rd workflow task must have a message with 1st update") + s.EqualValues(6, wt3.StartedEventId) + s.EqualValues(5, wt3.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4208,28 +4244,28 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, wt3.History) - // Now try to complete 2nd (speculative) WT, it should fail. - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt2.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, wt2.Messages[0]), - }) - s.Error(err, "Must fail because workflow task start time is different.") - s.IsType(&serviceerror.NotFound{}, err) - s.Contains(err.Error(), "Workflow task not found") + // Now try to complete 2nd (speculative) WT, it should fail. + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt2.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), wt2.Messages[0]), + }) + s.Error(err, "Must fail because workflow task start time is different.") + s.ErrorAs(err, new(*serviceerror.NotFound)) + s.Contains(err.Error(), "Workflow task not found") - // Try to complete 3rd WT, it should succeed - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt3.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, wt3.Messages[0]), - }) - s.NoError(err, "2nd speculative WT should be completed because it has same WT scheduled/started Id and startTime matches the accepted message is valid (same update Id)") + // Try to complete 3rd WT, it should succeed + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt3.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), wt3.Messages[0]), + }) + s.NoError(err, "2nd speculative WT should be completed because it has same WT scheduled/started Id and startTime matches the accepted message is valid (same update Id)") - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4240,54 +4276,54 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId":5} 9 WorkflowExecutionUpdateCompleted `, events) -} - -func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_NewWorkflowTaskWith2Updates() { - /* - Test scenario: - An update created a speculative WT and WT is dispatched to the worker (started). - Mutable state cleared, speculative WT and update registry are disappeared from server. - First update is retried and another update come in. - Second speculative WT is dispatched to worker with same WT scheduled/started Id but 2 updates. - The first speculative WT responds back, server rejected it (different start time). - The second speculative WT responds back, server accepted it. - */ - - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2) - - testCtx := testcore.NewContext() - - // Drain first WFT. - wt1, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), }) - s.NoError(err) - s.NotNil(wt1) - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt1.TaskToken, - }) - s.NoError(err) - // Send 1st update. It will create 2nd speculative WFT. - sendUpdateNoError(s, tv1) + t.Run("StaleSpeculativeWorkflowTask_Fail_NewWorkflowTaskWith2Updates", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + /* + Test scenario: + An update created a speculative WT and WT is dispatched to the worker (started). + Mutable state cleared, speculative WT and update registry are disappeared from server. + First update is retried and another update come in. + Second speculative WT is dispatched to worker with same WT scheduled/started Id but 2 updates. + The first speculative WT responds back, server rejected it (different start time). + The second speculative WT responds back, server accepted it. + */ - // Poll 2nd speculative WFT with 1st update. - wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - }) - s.NoError(err) - s.NotNil(wt2) - s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") - s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") - s.EqualValues(6, wt2.StartedEventId) - s.EqualValues(5, wt2.Messages[0].GetEventId()) - s.EqualHistory(` + mustStartWorkflow(s, s.Tv()) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2) + + testCtx := testcore.NewContext() + + // Drain first WFT. + wt1, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt1) + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt1.TaskToken, + }) + s.NoError(err) + + // Send 1st update. It will create 2nd speculative WFT. + sendUpdateNoError(s, tv1) + + // Poll 2nd speculative WFT with 1st update. + wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt2) + s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") + s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") + s.EqualValues(6, wt2.StartedEventId) + s.EqualValues(5, wt2.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4295,29 +4331,29 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_NewWorkflowT 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, wt2.History) - // Clear update registry. Speculative WFT disappears from server. - s.clearUpdateRegistryAndAbortPendingUpdates(tv) + // Clear update registry. Speculative WFT disappears from server. + clearUpdateRegistryAndAbortPendingUpdates(s, s.Tv()) - // Make sure UpdateWorkflowExecution call for the update "1" is retried and new (3rd) WFT is created as speculative with updateID=1. - waitUpdateAdmitted(s, tv1) + // Make sure UpdateWorkflowExecution call for the update "1" is retried and new (3rd) WFT is created as speculative with updateID=1. + waitUpdateAdmitted(s, tv1) - // Send 2nd update (with DIFFERENT updateId). It reuses already created 3rd WFT. - sendUpdateNoError(s, tv2) - // updateID=1 is still blocked. There must be 2 blocked updates now. + // Send 2nd update (with DIFFERENT updateId). It reuses already created 3rd WFT. + sendUpdateNoError(s, tv2) + // updateID=1 is still blocked. There must be 2 blocked updates now. - // Poll the 3rd speculative WFT. - wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - }) - s.NoError(err) - s.NotNil(wt3) - s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") - s.Len(wt3.Messages, 2, "3rd workflow task must have a message with 1st and 2nd updates") - s.EqualValues(6, wt3.StartedEventId) - s.EqualValues(5, wt3.Messages[0].GetEventId()) - s.EqualValues(5, wt3.Messages[1].GetEventId()) - s.EqualHistory(` + // Poll the 3rd speculative WFT. + wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt3) + s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") + s.Len(wt3.Messages, 2, "3rd workflow task must have a message with 1st and 2nd updates") + s.EqualValues(6, wt3.StartedEventId) + s.EqualValues(5, wt3.Messages[0].GetEventId()) + s.EqualValues(5, wt3.Messages[1].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4325,35 +4361,35 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_NewWorkflowT 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, wt3.History) - // Now try to complete 2nd speculative WT, it should fail because start time does not match. - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt2.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv1), - Messages: s.UpdateAcceptCompleteMessages(tv1, wt2.Messages[0]), - ReturnNewWorkflowTask: true, - }) - s.Error(err, "Must fail because start time is different.") - s.Contains(err.Error(), "Workflow task not found") - s.IsType(&serviceerror.NotFound{}, err) + // Now try to complete 2nd speculative WT, it should fail because start time does not match. + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt2.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(tv1), + Messages: s.UpdateAcceptCompleteMessages(tv1, wt2.Messages[0]), + ReturnNewWorkflowTask: true, + }) + s.Error(err, "Must fail because start time is different.") + s.Contains(err.Error(), "Workflow task not found") + s.ErrorAs(err, new(*serviceerror.NotFound)) - // Complete of the 3rd WT should succeed. It must accept both updates. - wt4Resp, err := s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt3.TaskToken, - Commands: append( - s.UpdateAcceptCompleteCommands(tv1), - s.UpdateAcceptCompleteCommands(tv2)...), - Messages: append( - s.UpdateAcceptCompleteMessages(tv1, wt3.Messages[0]), - s.UpdateAcceptCompleteMessages(tv2, wt3.Messages[1])...), - ReturnNewWorkflowTask: true, - }) - s.NoError(err) - s.NotNil(wt4Resp) + // Complete of the 3rd WT should succeed. It must accept both updates. + wt4Resp, err := s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt3.TaskToken, + Commands: append( + s.UpdateAcceptCompleteCommands(tv1), + s.UpdateAcceptCompleteCommands(tv2)...), + Messages: append( + s.UpdateAcceptCompleteMessages(tv1, wt3.Messages[0]), + s.UpdateAcceptCompleteMessages(tv2, wt3.Messages[1])...), + ReturnNewWorkflowTask: true, + }) + s.NoError(err) + s.NotNil(wt4Resp) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4366,26 +4402,25 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_NewWorkflowT 10 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} 11 WorkflowExecutionUpdateCompleted `, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_WorkerSkippedProcessing_RejectByServer() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_WorkerSkippedProcessing_RejectByServer", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2) - mustStartWorkflow(s, tv) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2) + var update2ResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - var update2ResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4393,91 +4428,92 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_WorkerSkippedProcessin 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted `, task.History) - update2ResultCh = sendUpdateNoError(s, tv2) - return nil, nil - case 3: - s.EqualHistory(` + update2ResultCh = sendUpdateNoError(s, tv2) + return nil, nil + case 3: + s.EqualHistory(` 4 WorkflowTaskCompleted // Speculative WT was dropped and history starts from 4 again. 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, task.History) - commands := append(s.UpdateAcceptCompleteCommands(tv2), - &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }) - return commands, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + commands := append(s.UpdateAcceptCompleteCommands(tv2), + &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }) + return commands, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.EqualValues(5, updRequestMsg.GetEventId()) - // Don't process update in WT. - return nil, nil - case 3: - s.Len(task.Messages, 1) - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + // Don't process update in WT. + return nil, nil + case 3: + s.Len(task.Messages, 1) + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv2, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.EqualValues(5, updRequestMsg.GetEventId()) + return s.UpdateAcceptCompleteMessages(tv2, updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Identity: "old_worker", - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Identity: "old_worker", + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv1) + updateResultCh := sendUpdateNoError(s, tv1) - // Process 2nd WT which ignores update message. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.Equal("Workflow Update is rejected because it wasn't processed by worker. Probably, Workflow Update is not supported by the worker.", updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(3, updateResp.ResetHistoryEventId) + // Process 2nd WT which ignores update message. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("Workflow Update is rejected because it wasn't processed by worker. Probably, Workflow Update is not supported by the worker.", updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(3, updateResp.ResetHistoryEventId) - // Process 3rd WT which completes 2nd update and workflow. - update2Resp, err := poller.HandlePartialWorkflowTask(updateResp.GetWorkflowTask(), false) - s.NoError(err) - s.NotNil(update2Resp) - update2Result := <-update2ResultCh - s.EqualValues("success-result-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), update2Result.GetOutcome().GetSuccess())) + // Process 3rd WT which completes 2nd update and workflow. + update2Resp, err := poller.HandlePartialWorkflowTask(updateResp.GetWorkflowTask(), false) + s.NoError(err) + s.NotNil(update2Resp) + update2Result := <-update2ResultCh + s.Equal("success-result-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), update2Result.GetOutcome().GetSuccess())) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4488,66 +4524,66 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_WorkerSkippedProcessin 8 WorkflowExecutionUpdateAccepted 9 WorkflowExecutionUpdateCompleted 10 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestLastWorkflowTask_HasUpdateMessage() { - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) + t.Run("LastWorkflowTask_HasUpdateMessage", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - completeWorkflowCommand := &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ - Result: tv.Any().Payloads(), + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + completeWorkflowCommand := &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ + Result: s.Tv().Any().Payloads(), + }, }, - }, - } - return append(s.UpdateAcceptCommands(tv), completeWorkflowCommand), nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - return s.UpdateAcceptMessages(tv, task.Messages[0]), nil - }, - Logger: s.Logger, - T: s.T(), - } + } + return append(s.UpdateAcceptCommands(s.Tv()), completeWorkflowCommand), nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + return s.UpdateAcceptMessages(s.Tv(), task.Messages[0]), nil + }, + Logger: s.Logger, + T: s.T(), + } - updateResultCh := sendUpdateNoErrorWaitPolicyAccepted(s, tv) - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResult := <-updateResultCh - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult.GetStage()) - s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult.GetOutcome().GetFailure().GetMessage()) + updateResultCh := sendUpdateNoErrorWaitPolicyAccepted(s, s.Tv()) + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResult := <-updateResultCh + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult.GetStage()) + s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowExecutionUpdateAccepted 6 WorkflowExecutionCompleted - `, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) -} + `, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_QueryFailureClearsWFContext() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_QueryFailureClearsWFContext", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4555,343 +4591,347 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_QueryFailureClearsWFCo 5 WorkflowTaskScheduled 6 WorkflowTaskStarted `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - type QueryResult struct { - Resp *workflowservice.QueryWorkflowResponse - Err error - } - queryFn := func(resCh chan<- QueryResult) { - // There is no query handler, and query timeout is ok for this test. - // But first query must not time out before 2nd query reached server, - // because 2 queries overflow the query buffer (default size 1), - // which leads to clearing of WF context. - shortCtx, cancel := context.WithTimeout(testcore.NewContext(), 100*time.Millisecond) - defer cancel() - queryResp, err := s.FrontendClient().QueryWorkflow(shortCtx, &workflowservice.QueryWorkflowRequest{ - Namespace: s.Namespace().String(), - Execution: tv.WorkflowExecution(), - Query: &querypb.WorkflowQuery{ - QueryType: tv.Any().String(), - }, - }) - resCh <- QueryResult{Resp: queryResp, Err: err} - } + type QueryResult struct { + Resp *workflowservice.QueryWorkflowResponse + Err error + } + queryFn := func(resCh chan<- QueryResult) { + // There is no query handler, and query timeout is ok for this test. + // But first query must not time out before 2nd query reached server, + // because 2 queries overflow the query buffer (default size 1), + // which leads to clearing of WF context. + shortCtx, cancel := context.WithTimeout(testcore.NewContext(), 100*time.Millisecond) + defer cancel() + queryResp, err := s.FrontendClient().QueryWorkflow(shortCtx, &workflowservice.QueryWorkflowRequest{ + Namespace: s.Namespace().String(), + Execution: s.Tv().WorkflowExecution(), + Query: &querypb.WorkflowQuery{ + QueryType: s.Tv().Any().String(), + }, + }) + resCh <- QueryResult{Resp: queryResp, Err: err} + } - query1ResultCh := make(chan QueryResult) - query2ResultCh := make(chan QueryResult) - go queryFn(query1ResultCh) - go queryFn(query2ResultCh) - query1Res := <-query1ResultCh - query2Res := <-query2ResultCh - s.Error(query1Res.Err) - s.Error(query2Res.Err) - s.Nil(query1Res.Resp) - s.Nil(query2Res.Resp) + query1ResultCh := make(chan QueryResult) + query2ResultCh := make(chan QueryResult) + go queryFn(query1ResultCh) + go queryFn(query2ResultCh) + query1Res := <-query1ResultCh + query2Res := <-query2ResultCh + s.Error(query1Res.Err) + s.Error(query2Res.Err) + s.Nil(query1Res.Resp) + s.Nil(query2Res.Resp) - var queryBufferFullErr *serviceerror.ResourceExhausted - if common.IsContextDeadlineExceededErr(query1Res.Err) { - s.True(common.IsContextDeadlineExceededErr(query1Res.Err), "one of query errors must be CDE") - s.ErrorAs(query2Res.Err, &queryBufferFullErr, "one of query errors must `query buffer is full`") - s.Contains(query2Res.Err.Error(), "query buffer is full", "one of query errors must `query buffer is full`") - } else { - s.ErrorAs(query1Res.Err, &queryBufferFullErr, "one of query errors must `query buffer is full`") - s.Contains(query1Res.Err.Error(), "query buffer is full", "one of query errors must `query buffer is full`") - s.True(common.IsContextDeadlineExceededErr(query2Res.Err), "one of query errors must be CDE") - } + var queryBufferFullErr *serviceerror.ResourceExhausted + if common.IsContextDeadlineExceededErr(query1Res.Err) { + s.True(common.IsContextDeadlineExceededErr(query1Res.Err), "one of query errors must be CDE") + s.ErrorAs(query2Res.Err, &queryBufferFullErr, "one of query errors must `query buffer is full`") + s.Contains(query2Res.Err.Error(), "query buffer is full", "one of query errors must `query buffer is full`") + } else { + s.ErrorAs(query1Res.Err, &queryBufferFullErr, "one of query errors must `query buffer is full`") + s.Contains(query1Res.Err.Error(), "query buffer is full", "one of query errors must `query buffer is full`") + s.True(common.IsContextDeadlineExceededErr(query2Res.Err), "one of query errors must be CDE") + } - // "query buffer is full" error clears WF context. If update registry is not cleared together with context (old behaviour), - // then update stays there but speculative WFT which supposed to deliver it, is cleared. - // Subsequent retry attempts of "UpdateWorkflowExecution" API wouldn't help, because update is deduped by registry, - // and new WFT is not created. Update is not delivered to the worker until new WFT is created. - // If registry is cleared together with WF context (current behaviour), retries of "UpdateWorkflowExecution" - // will create new update and WFT. + // "query buffer is full" error clears WF context. If update registry is not cleared together with context (old behaviour), + // then update stays there but speculative WFT which supposed to deliver it, is cleared. + // Subsequent retry attempts of "UpdateWorkflowExecution" API wouldn't help, because update is deduped by registry, + // and new WFT is not created. Update is not delivered to the worker until new WFT is created. + // If registry is cleared together with WF context (current behaviour), retries of "UpdateWorkflowExecution" + // will create new update and WFT. - // Wait to make sure that UpdateWorkflowExecution call is retried, update and speculative WFT are recreated. - time.Sleep(500 * time.Millisecond) //nolint:forbidigo + // Wait to make sure that UpdateWorkflowExecution call is retried, update and speculative WFT are recreated. + time.Sleep(500 * time.Millisecond) //nolint:forbidigo - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, updateResp.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, updateResp.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Was speculative WT... 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted // ...and events were written to the history when WT completes. + 7 WorkflowTaskCompleted // ...and events were written to the history when WT completes. 8 WorkflowExecutionUpdateAccepted 9 WorkflowExecutionUpdateCompleted `, events) -} + }) -func (s *UpdateWorkflowSuite) TestUpdatesAreSentToWorkerInOrderOfAdmission() { - // If our implementation is not in fact ordering updates correctly, then it may be ordering them - // non-deterministically. This number should be high enough that the false-negative rate of the test is low, but - // must not exceed our limit on number of in-flight updates. If we were picking a random ordering then the - // false-negative rate would be 1/(nUpdates!). - nUpdates := 10 + t.Run("UpdatesAreSentToWorkerInOrderOfAdmission", func(t *testing.T) { + s := testcore.NewEnv(t) + // If our implementation is not in fact ordering updates correctly, then it may be ordering them + // non-deterministically. This number should be high enough that the false-negative rate of the test is low, but + // must not exceed our limit on number of in-flight updates. If we were picking a random ordering then the + // false-negative rate would be 1/(nUpdates!). + nUpdates := 10 - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - for i := 0; i < nUpdates; i++ { - // Sequentially send updates one by one. - sendUpdateNoError(s, tv.WithUpdateIDNumber(i)) - } + mustStartWorkflow(s, s.Tv()) + for i := 0; i < nUpdates; i++ { + // Sequentially send updates one by one. + sendUpdateNoError(s, s.Tv().WithUpdateIDNumber(i)) + } - wtHandlerCalls := 0 - msgHandlerCalls := 0 - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - msgHandlerCalls++ - var commands []*commandpb.Command - for i := range task.Messages { - commands = append(commands, s.UpdateAcceptCompleteCommands(tv.WithMessageIDNumber(i))...) - } - return commands, nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - wtHandlerCalls++ - s.Len(task.Messages, nUpdates) - var messages []*protocolpb.Message - // Updates were sent in sequential order of updateId => messages must be ordered in the same way. - for i, m := range task.Messages { - s.Equal(tv.WithUpdateIDNumber(i).UpdateID(), m.ProtocolInstanceId) - messages = append(messages, s.UpdateAcceptCompleteMessages(tv.WithMessageIDNumber(i), m)...) - } - return messages, nil - }, - Logger: s.Logger, - T: s.T(), - } - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.Equal(1, wtHandlerCalls) - s.Equal(1, msgHandlerCalls) + wtHandlerCalls := 0 + msgHandlerCalls := 0 + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + msgHandlerCalls++ + var commands []*commandpb.Command + for i := range task.Messages { + commands = append(commands, s.UpdateAcceptCompleteCommands(s.Tv().WithMessageIDNumber(i))...) + } + return commands, nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + wtHandlerCalls++ + s.Len(task.Messages, nUpdates) + var messages []*protocolpb.Message + // Updates were sent in sequential order of updateId => messages must be ordered in the same way. + for i, m := range task.Messages { + s.Equal(s.Tv().WithUpdateIDNumber(i).UpdateID(), m.ProtocolInstanceId) + messages = append(messages, s.UpdateAcceptCompleteMessages(s.Tv().WithMessageIDNumber(i), m)...) + } + return messages, nil + }, + Logger: s.Logger, + T: s.T(), + } + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.Equal(1, wtHandlerCalls) + s.Equal(1, msgHandlerCalls) - expectedHistory := ` + expectedHistory := ` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted ` - for i := 0; i < nUpdates; i++ { - tvi := tv.WithUpdateIDNumber(i) - expectedHistory += fmt.Sprintf(` + for i := 0; i < nUpdates; i++ { + tvi := s.Tv().WithUpdateIDNumber(i) + expectedHistory += fmt.Sprintf(` %d WorkflowExecutionUpdateAccepted {"AcceptedRequest":{"Meta": {"UpdateId": "%s"}}} %d WorkflowExecutionUpdateCompleted {"Meta": {"UpdateId": "%s"}}`, - 5+2*i, tvi.UpdateID(), - 6+2*i, tvi.UpdateID()) - } + 5+2*i, tvi.UpdateID(), + 6+2*i, tvi.UpdateID()) + } - history := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(expectedHistory, history) -} + history := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(expectedHistory, history) + }) -func (s *UpdateWorkflowSuite) TestWaitAccepted_GotCompleted() { - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) + t.Run("WaitAccepted_GotCompleted", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - return s.UpdateAcceptCompleteCommands(tv), nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - return s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), nil - }, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + return s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), nil + }, + Logger: s.Logger, + T: s.T(), + } - // Send Update with intent to wait for Accepted stage only, - updateResultCh := sendUpdateNoErrorWaitPolicyAccepted(s, tv) - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResult := <-updateResultCh - // but Update was accepted and completed on the same WFT, and outcome was returned. - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult.GetStage()) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + // Send Update with intent to wait for Accepted stage only, + updateResultCh := sendUpdateNoErrorWaitPolicyAccepted(s, s.Tv()) + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResult := <-updateResultCh + // but Update was accepted and completed on the same WFT, and outcome was returned. + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult.GetStage()) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowExecutionUpdateAccepted 6 WorkflowExecutionUpdateCompleted - `, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) -} + `, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) -func (s *UpdateWorkflowSuite) TestContinueAsNew_UpdateIsNotCarriedOver() { - tv := testvars.New(s.T()) - firstRunID := mustStartWorkflow(s, tv) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2) + t.Run("ContinueAsNew_UpdateIsNotCarriedOver", func(t *testing.T) { + s := testcore.NewEnv(t) + firstRunID := mustStartWorkflow(s, s.Tv()) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2) - /* - 1st Update goes to the 1st run and accepted (but not completed) by Workflow. - While this WFT is running, 2nd Update is sent, and WFT is completing with CAN for the 1st run. - There are 2 Updates in the registry of the 1st run: 1st is accepted and 2nd is admitted. - Both of them are aborted but with different errors: - - Admitted Update is aborted with retryable "workflow is closing" error. SDK should retry this error - and new attempt should land on the new run. - - Accepted Update is aborted with update failure. - */ + /* + 1st Update goes to the 1st run and accepted (but not completed) by Workflow. + While this WFT is running, 2nd Update is sent, and WFT is completing with CAN for the 1st run. + There are 2 Updates in the registry of the 1st run: 1st is accepted and 2nd is admitted. + Both of them are aborted but with different errors: + - Admitted Update is aborted with retryable "workflow is closing" error. SDK should retry this error + and new attempt should land on the new run. + - Accepted Update is aborted with update failure. + */ - var update2ResponseCh <-chan updateResponseErr + var update2ResponseCh <-chan updateResponseErr - poller1 := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - // Send 2nd Update while WFT is running. - update2ResponseCh = sendUpdate(context.Background(), s, tv2) - canCommand := &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.WithTaskQueueNumber(2).TaskQueue(), - }}, - } - return append(s.UpdateAcceptCommands(tv1), canCommand), nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - return s.UpdateAcceptMessages(tv1, task.Messages[0]), nil - }, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller1 := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + // Send 2nd Update while WFT is running. + update2ResponseCh = sendUpdate(context.Background(), s, tv2) + canCommand := &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: s.Tv().WorkflowType(), + TaskQueue: s.Tv().WithTaskQueueNumber(2).TaskQueue(), + }}, + } + return append(s.UpdateAcceptCommands(tv1), canCommand), nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + return s.UpdateAcceptMessages(tv1, task.Messages[0]), nil + }, + Logger: s.Logger, + T: s.T(), + } - poller2 := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.WithTaskQueueNumber(2).TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - return nil, nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - s.Empty(task.Messages, "no Updates should be carried over to the 2nd run") - return nil, nil - }, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller2 := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().WithTaskQueueNumber(2).TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + return nil, nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + s.Empty(task.Messages, "no Updates should be carried over to the 2nd run") + return nil, nil + }, + Logger: s.Logger, + T: s.T(), + } - update1ResponseCh := sendUpdate(context.Background(), s, tv1) - _, err := poller1.PollAndProcessWorkflowTask() - s.NoError(err) + update1ResponseCh := sendUpdate(context.Background(), s, tv1) + _, err := poller1.PollAndProcessWorkflowTask() + s.NoError(err) - _, err = poller2.PollAndProcessWorkflowTask() - s.NoError(err) + _, err = poller2.PollAndProcessWorkflowTask() + s.NoError(err) - update1Response := <-update1ResponseCh - s.NoError(update1Response.err) - s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", update1Response.response.GetOutcome().GetFailure().GetMessage()) + update1Response := <-update1ResponseCh + s.NoError(update1Response.err) + s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", update1Response.response.GetOutcome().GetFailure().GetMessage()) - update2Response := <-update2ResponseCh - s.Error(update2Response.err) - var resourceExhausted *serviceerror.ResourceExhausted - s.ErrorAs(update2Response.err, &resourceExhausted) - s.Equal("workflow operation can not be applied because workflow is closing", update2Response.err.Error()) + update2Response := <-update2ResponseCh + s.Error(update2Response.err) + var resourceExhausted *serviceerror.ResourceExhausted + s.ErrorAs(update2Response.err, &resourceExhausted) + s.Equal("workflow operation can not be applied because workflow is closing", update2Response.err.Error()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionContinuedAsNew`, s.GetHistory(s.Namespace().String(), tv.WithRunID(firstRunID).WorkflowExecution())) + 6 WorkflowExecutionContinuedAsNew`, s.GetHistory(s.Namespace().String(), s.Tv().WithRunID(firstRunID).WorkflowExecution())) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted`, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) -} + 4 WorkflowTaskCompleted`, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) -func (s *UpdateWorkflowSuite) TestContinueAsNew_Suggestion() { - // setup CAN suggestion to be at 2nd Update - cleanup1 := s.OverrideDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdates, 3) - defer cleanup1() - cleanup2 := s.OverrideDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdatesSuggestContinueAsNewThreshold, 0.5) - defer cleanup2() + t.Run("ContinueAsNew_Suggestion", func(t *testing.T) { + // setup CAN suggestion to be at 2nd Update + s := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdates, 3), + testcore.WithDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdatesSuggestContinueAsNewThreshold, 0.5), + ) - // start workflow - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + // start workflow + mustStartWorkflow(s, s.Tv()) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) - // send Update #1 - no CAN suggested - tv1 := tv.WithUpdateIDNumber(1) - updateResultCh := sendUpdateNoError(s, tv1) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv1, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistoryEvents(` + // send Update #1 - no CAN suggested + tv1 := s.Tv().WithUpdateIDNumber(1) + updateResultCh := sendUpdateNoError(s, tv1) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv1, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted {"SuggestContinueAsNew": false} @@ -4899,154 +4939,294 @@ func (s *UpdateWorkflowSuite) TestContinueAsNew_Suggestion() { 5 WorkflowTaskScheduled 6 WorkflowTaskStarted {"SuggestContinueAsNew": false}`, task.History.Events) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv1, task.Messages[0]), - }, nil - }) - s.NoError(err) - <-updateResultCh + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(tv1, task.Messages[0]), + }, nil + }) + s.NoError(err) + <-updateResultCh - // send Update #2 - CAN suggested - tv2 := tv.WithUpdateIDNumber(2) - updateResultCh = sendUpdateNoError(s, tv2) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv2, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistoryEventsSuffix(` + // send Update #2 - CAN suggested + tv2 := s.Tv().WithUpdateIDNumber(2) + updateResultCh = sendUpdateNoError(s, tv2) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv2, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistoryEventsSuffix(` WorkflowTaskStarted {"SuggestContinueAsNew": true}`, task.History.Events) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv2, task.Messages[0]), - }, nil - }) - s.NoError(err) - <-updateResultCh -} + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(tv2, task.Messages[0]), + }, nil + }) + s.NoError(err) + <-updateResultCh + }) -func (s *UpdateWorkflowSuite) TestUpdateWithStart() { - type multiopsResponseErr struct { - response *workflowservice.ExecuteMultiOperationResponse - err error - } - - sendUpdateWithStart := func( - ctx context.Context, - startReq *workflowservice.StartWorkflowExecutionRequest, - updateReq *workflowservice.UpdateWorkflowExecutionRequest, - ) chan multiopsResponseErr { - capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() - defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - - retCh := make(chan multiopsResponseErr) - go func() { - resp, err := s.FrontendClient().ExecuteMultiOperation( - ctx, - &workflowservice.ExecuteMultiOperationRequest{ - Namespace: s.Namespace().String(), - Operations: []*workflowservice.ExecuteMultiOperationRequest_Operation{ - { - Operation: &workflowservice.ExecuteMultiOperationRequest_Operation_StartWorkflow{ - StartWorkflow: startReq, - }, - }, - { - Operation: &workflowservice.ExecuteMultiOperationRequest_Operation_UpdateWorkflow{ - UpdateWorkflow: updateReq, - }, - }, - }, - }) - - if err == nil { - s.Len(resp.Responses, 2) - - startRes := resp.Responses[0].Response.(*workflowservice.ExecuteMultiOperationResponse_Response_StartWorkflow).StartWorkflow - s.NotZero(startRes.RunId) - - updateRes := resp.Responses[1].Response.(*workflowservice.ExecuteMultiOperationResponse_Response_UpdateWorkflow).UpdateWorkflow - if updateReq.WaitPolicy.LifecycleStage == enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED { - s.NotNil(updateRes.Outcome) - s.NotZero(updateRes.Outcome.String()) - } - } - - // make sure there's no lock contention - s.Empty(capture.Snapshot()[metrics.TaskWorkflowBusyCounter.Name()]) - - retCh <- multiopsResponseErr{resp, err} - }() - return retCh - } - - startWorkflowReq := func(tv *testvars.TestVars) *workflowservice.StartWorkflowExecutionRequest { - return &workflowservice.StartWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowId: tv.WorkflowID(), - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), + t.Run("UpdateWithStart", func(t *testing.T) { + type multiopsResponseErr struct { + response *workflowservice.ExecuteMultiOperationResponse + err error } - } - s.Run("workflow is not running", func() { + sendUpdateWithStart := func(s testcore.Env, ctx context.Context, startReq *workflowservice.StartWorkflowExecutionRequest, updateReq *workflowservice.UpdateWorkflowExecutionRequest) chan multiopsResponseErr { + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - for _, p := range []enumspb.WorkflowIdConflictPolicy{ - enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - } { - s.Run(fmt.Sprintf("start workflow and send update (with conflict policy %v)", p), func() { + retCh := make(chan multiopsResponseErr) + go func() { + resp, err := s.FrontendClient().ExecuteMultiOperation( + ctx, + &workflowservice.ExecuteMultiOperationRequest{ + Namespace: s.Namespace().String(), + Operations: []*workflowservice.ExecuteMultiOperationRequest_Operation{ + { + Operation: &workflowservice.ExecuteMultiOperationRequest_Operation_StartWorkflow{ + StartWorkflow: startReq, + }, + }, + { + Operation: &workflowservice.ExecuteMultiOperationRequest_Operation_UpdateWorkflow{ + UpdateWorkflow: updateReq, + }, + }, + }, + }) - s.Run("and accept", func() { - tv := testvars.New(s.T()) + if err == nil { + // Use assert (not require) in goroutine - require calls t.FailNow() which panics + //nolint:testifylint // intentional use of assert in goroutine + assert.Len(s.T(), resp.Responses, 2) - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = p - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + startRes := resp.Responses[0].Response.(*workflowservice.ExecuteMultiOperationResponse_Response_StartWorkflow).StartWorkflow + assert.NotEmpty(s.T(), startRes.RunId) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) + updateRes := resp.Responses[1].Response.(*workflowservice.ExecuteMultiOperationResponse_Response_UpdateWorkflow).UpdateWorkflow + if updateReq.WaitPolicy.LifecycleStage == enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED { + assert.NotNil(s.T(), updateRes.Outcome) + assert.NotEmpty(s.T(), updateRes.Outcome.String()) + } + } - uwsRes := <-uwsCh - s.NoError(err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - requireStartedAndRunning(s.T(), startResp) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + // make sure there's no lock contention + //nolint:testifylint // intentional use of assert in goroutine + assert.Empty(s.T(), capture.Snapshot()[metrics.TaskWorkflowBusyCounter.Name()]) - // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) - s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) + retCh <- multiopsResponseErr{resp, err} + }() + return retCh + } - s.EqualHistoryEvents(` + startWorkflowReq := func(s testcore.Env, tv *testvars.TestVars) *workflowservice.StartWorkflowExecutionRequest { + return &workflowservice.StartWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowId: tv.WorkflowID(), + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + Identity: tv.WorkerIdentity(), + } + } + + t.Run("workflow is not running", func(t *testing.T) { + for _, p := range []enumspb.WorkflowIdConflictPolicy{ + enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + } { + t.Run(fmt.Sprintf("start workflow and send update (with conflict policy %v)", p), func(t *testing.T) { + t.Run("and accept", func(t *testing.T) { + s := testcore.NewEnv(t) + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = p + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + requireStartedAndRunning(s.T(), startResp) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + + // poll update to ensure same outcome is returned + pollRes, err := pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.NoError(err) + s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) + + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionUpdateCompleted`, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + 6 WorkflowExecutionUpdateCompleted`, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) + + t.Run("and reject", func(t *testing.T) { + s := testcore.NewEnv(t) + startReq := startWorkflowReq(s, s.Tv()) + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + requireStartedAndRunning(s.T(), startResp) + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateRep.GetOutcome().GetFailure().GetMessage()) + + // poll update to ensure same outcome is returned + _, err = pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.Error(err) + s.ErrorAs(err, new(*serviceerror.NotFound)) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted`, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) }) + } + }) - s.Run("and reject", func() { - tv := testvars.New(s.T()) + t.Run("workflow is running", func(t *testing.T) { + t.Run("workflow id conflict policy use-existing: only send update", func(t *testing.T) { + t.Run("and accept", func(t *testing.T) { + s := testcore.NewEnv(t) + // start workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) - startReq := startWorkflowReq(tv) - updateReq := updateWorkflowRequest(s, tv, + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, task.Messages[0]), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + requireNotStartedButRunning(s.T(), startResp) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + + // poll update to ensure same outcome is returned + pollRes, err := pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.Nil(err) + s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled + 6 WorkflowTaskStarted + 7 WorkflowTaskCompleted + 8 WorkflowExecutionUpdateAccepted + 9 WorkflowExecutionUpdateCompleted`, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) + + t.Run("and reject", func(t *testing.T) { + s := testcore.NewEnv(t) + // start workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + requireNotStartedButRunning(s.T(), startResp) + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateRep.GetOutcome().GetFailure().GetMessage()) + + // poll update to ensure same outcome is returned + _, err = pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.Error(err) + s.ErrorAs(err, new(*serviceerror.NotFound)) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + `, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) + }) + + t.Run("workflow id conflict policy terminate-existing", func(t *testing.T) { + t.Run("terminate workflow first, then start and update", func(t *testing.T) { + s := testcore.NewEnv(t) + // start workflow + firstWF, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), }, nil }) s.NoError(err) @@ -5056,148 +5236,206 @@ func (s *UpdateWorkflowSuite) TestUpdateWithStart() { startResp := uwsRes.response.Responses[0].GetStartWorkflow() updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() requireStartedAndRunning(s.T(), startResp) - s.Equal("rejection-of-"+tv.UpdateID(), updateRep.GetOutcome().GetFailure().GetMessage()) + s.Equal(startResp.RunId, updateRep.UpdateRef.GetWorkflowExecution().RunId) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + + // ensure workflow was terminated + descResp, err := s.FrontendClient().DescribeWorkflowExecution(testcore.NewContext(), + &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: startReq.WorkflowId, RunId: firstWF.RunId}, + }) + s.NoError(err) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, descResp.WorkflowExecutionInfo.Status) // poll update to ensure same outcome is returned - _, err = pollUpdate(s, tv, + pollRes, err := pollUpdate(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Error(err) - s.IsType(&serviceerror.NotFound{}, err) + s.NoError(err) + s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) + }) - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted`, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + t.Run("given an accepted update, attach to it", func(t *testing.T) { + s := testcore.NewEnv(t) + // 1st update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh1 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + // accept the update + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes1 := <-uwsCh1 + s.NoError(uwsRes1.err) + startResp1 := uwsRes1.response.Responses[0].GetStartWorkflow() + updateRep1 := uwsRes1.response.Responses[1].GetUpdateWorkflow() + s.True(startResp1.Started) + s.Equal(startResp1.RunId, updateRep1.UpdateRef.GetWorkflowExecution().RunId) + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED, updateRep1.Stage) + + // 2nd update-with-start: attaches to update instead of terminating workflow + uwsCh2 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + uwsRes2 := <-uwsCh2 + s.NoError(uwsRes2.err) + startResp2 := uwsRes2.response.Responses[0].GetStartWorkflow() + updateRep2 := uwsRes2.response.Responses[1].GetUpdateWorkflow() + s.False(startResp2.Started) + s.Equal(startResp2.RunId, startResp1.RunId) // no termination + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED, updateRep2.Stage) }) }) - } - }) - s.Run("workflow is running", func() { - - s.Run("workflow id conflict policy use-existing: only send update", func() { - - s.Run("and accept", func() { - tv := testvars.New(s.T()) - - // start workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) + t.Run("workflow id conflict policy fail: abort multi operation", func(t *testing.T) { + s := testcore.NewEnv(t) + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) s.NoError(err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) + // start workflow + startWorkflowReq(s, s.Tv()) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) s.NoError(err) // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL + updateReq := updateWorkflowRequest(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) uwsRes := <-uwsCh - s.NoError(uwsRes.err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - requireNotStartedButRunning(s.T(), startResp) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) - - // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) - s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled - 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted - 8 WorkflowExecutionUpdateAccepted - 9 WorkflowExecutionUpdateCompleted`, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + s.Error(uwsRes.err) + s.Equal("Update-with-Start could not be executed.", uwsRes.err.Error()) + errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted + s.ErrorAs(errs[0], &alreadyStartedErr) + s.Equal("Operation was aborted.", errs[1].Error()) }) - s.Run("and reject", func() { - tv := testvars.New(s.T()) + t.Run("receive completed update result", func(t *testing.T) { + _ = testcore.NewEnv(t) // unused s + for _, p := range []enumspb.WorkflowIdConflictPolicy{ + enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + } { + t.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func(t *testing.T) { + s := testcore.NewEnv(t) + startReq := startWorkflowReq(s, s.Tv()) + updReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - // start workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) + // 1st update-with-start + uwsCh1 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updReq) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + uwsRes1 := <-uwsCh1 + s.NoError(uwsRes1.err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + // 2nd update-with-start: using *same* UpdateID - but *different* RequestID + uwsRes2 := <-sendUpdateWithStart(s, testcore.NewContext(), startReq, updReq) + s.NoError(uwsRes2.err) - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + s.Equal(uwsRes1.response.Responses[0].GetStartWorkflow().RunId, uwsRes2.response.Responses[0].GetStartWorkflow().RunId) + s.Equal(uwsRes1.response.Responses[1].GetUpdateWorkflow().Outcome.String(), uwsRes2.response.Responses[1].GetUpdateWorkflow().Outcome.String()) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, task.Messages[0]), - }, nil + // poll update to ensure same outcome is returned + pollRes, err := pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.NoError(err) + s.Equal(uwsRes1.response.Responses[1].GetUpdateWorkflow().Outcome.String(), pollRes.Outcome.String()) }) - s.NoError(err) + } + }) - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - requireNotStartedButRunning(s.T(), startResp) - s.Equal("rejection-of-"+tv.UpdateID(), updateRep.GetOutcome().GetFailure().GetMessage()) + t.Run("dedupes start", func(t *testing.T) { + _ = testcore.NewEnv(t) // unused s + for _, p := range []enumspb.WorkflowIdConflictPolicy{ + enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + } { + t.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func(t *testing.T) { + s := testcore.NewEnv(t) + startReq := startWorkflowReq(s, s.Tv()) + startReq.RequestId = "request_id" + startReq.WorkflowIdConflictPolicy = p + updReq1 := updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(1), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - // poll update to ensure same outcome is returned - _, err = pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Error(err) - s.IsType(&serviceerror.NotFound{}, err) + // 1st update-with-start + uwsCh1 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updReq1) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + uwsRes1 := <-uwsCh1 + s.NoError(uwsRes1.err) - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - `, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + // 2nd update-with-start: using *same* RequestID - but *different* UpdateID + updReq2 := updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(2), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh2 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updReq2) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + uwsRes2 := <-uwsCh2 + s.NoError(uwsRes1.err) + + s.Equal(uwsRes1.response.Responses[0].GetStartWorkflow().RunId, uwsRes2.response.Responses[0].GetStartWorkflow().RunId) + }) + } }) }) - s.Run("workflow id conflict policy terminate-existing", func() { - - s.Run("terminate workflow first, then start and update", func() { - tv := testvars.New(s.T()) - - // start workflow - firstWF, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) + t.Run("workflow is closed", func(t *testing.T) { + t.Run("workflow id reuse policy allow-duplicate", func(t *testing.T) { + s := testcore.NewEnv(t) + // start and terminate workflow + initialWorkflow, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) s.NoError(err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), + &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) s.NoError(err) // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING - updateReq := updateWorkflowRequest(s, tv, + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE + updateReq := updateWorkflowRequest(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), }, nil }) s.NoError(err) @@ -5207,438 +5445,166 @@ func (s *UpdateWorkflowSuite) TestUpdateWithStart() { startResp := uwsRes.response.Responses[0].GetStartWorkflow() updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() requireStartedAndRunning(s.T(), startResp) - s.Equal(startResp.RunId, updateRep.UpdateRef.GetWorkflowExecution().RunId) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) - // ensure workflow was terminated - descResp, err := s.FrontendClient().DescribeWorkflowExecution(testcore.NewContext(), - &workflowservice.DescribeWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - Execution: &commonpb.WorkflowExecution{WorkflowId: startReq.WorkflowId, RunId: firstWF.RunId}, - }) - s.NoError(err) - s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, descResp.WorkflowExecutionInfo.Status) + // ensure terminated workflow is not locked by update-with-start + err = s.SendSignal(s.Namespace().String(), &commonpb.WorkflowExecution{ + WorkflowId: s.Tv().WorkflowID(), + RunId: initialWorkflow.RunId, + }, s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.ErrorContains(err, "workflow execution already completed") // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, + pollRes, err := pollUpdate(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) + s.NoError(err) s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) }) - s.Run("given an accepted update, attach to it", func() { - tv := testvars.New(s.T()) + t.Run("workflow id reuse policy reject-duplicate", func(t *testing.T) { + s := testcore.NewEnv(t) + // start and terminate workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) - // 1st update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh1 := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) - // accept the update - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, + _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), + &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + uwsRes := <-uwsCh + s.Error(uwsRes.err) + s.Equal("Update-with-Start could not be executed.", uwsRes.err.Error()) + errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + s.Contains(errs[0].Error(), "Workflow execution already finished") + var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted + s.ErrorAs(errs[0], &alreadyStartedErr) + s.Equal("Operation was aborted.", errs[1].Error()) + }) + + t.Run("receive completed update result", func(t *testing.T) { + _ = testcore.NewEnv(t) // unused s + for _, p := range []enumspb.WorkflowIdConflictPolicy{ + enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + } { + t.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func(t *testing.T) { + s := testcore.NewEnv(t) + // 1st update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = p + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + startResp1 := uwsRes.response.Responses[0].GetStartWorkflow() + _ = uwsRes.response.Responses[1].GetUpdateWorkflow() + requireStartedAndRunning(s.T(), startResp1) + + // terminate workflow + _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), + &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) + s.NoError(err) + + // 2nd update-with-start (using the same Update ID but different Request ID) + uwsRes = <-sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + s.NoError(uwsRes.err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + s.False(startResp.Started) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, startResp.Status) + // TODO: check startResp.Running + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + }) + } + }) + }) + + t.Run("workflow start conflict", func(t *testing.T) { + t.Run("workflow id conflict policy fail: use-existing", func(t *testing.T) { + // Uses InjectHook which requires a dedicated cluster to avoid conflicts with other tests. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + + // simulate a race condition + s.InjectHook(testhooks.UpdateWithStartInBetweenLockAndStart, func() { + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startReq) + s.NoError(err) + }) + + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{}, nil + }) + s.NoError(err) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptMessages(tv, task.Messages[0]), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), }, nil }) s.NoError(err) - uwsRes1 := <-uwsCh1 - s.NoError(uwsRes1.err) - startResp1 := uwsRes1.response.Responses[0].GetStartWorkflow() - updateRep1 := uwsRes1.response.Responses[1].GetUpdateWorkflow() - s.True(startResp1.Started) - s.Equal(startResp1.RunId, updateRep1.UpdateRef.GetWorkflowExecution().RunId) - s.Equal(updateRep1.Stage, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED) - - // 2nd update-with-start: attaches to update instead of terminating workflow - uwsCh2 := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - uwsRes2 := <-uwsCh2 - s.NoError(uwsRes2.err) - startResp2 := uwsRes2.response.Responses[0].GetStartWorkflow() - updateRep2 := uwsRes2.response.Responses[1].GetUpdateWorkflow() - s.False(startResp2.Started) - s.Equal(startResp2.RunId, startResp1.RunId) // no termination - s.Equal(updateRep2.Stage, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED) + <-uwsCh }) }) - s.Run("workflow id conflict policy fail: abort multi operation", func() { - tv := testvars.New(s.T()) - - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - - // start workflow - startWorkflowReq(tv) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL - updateReq := updateWorkflowRequest(s, tv, &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - uwsRes := <-uwsCh - s.Error(uwsRes.err) - s.Equal("Update-with-Start could not be executed.", uwsRes.err.Error()) - errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted - s.ErrorAs(errs[0], &alreadyStartedErr) - s.Equal("Operation was aborted.", errs[1].Error()) - }) - - s.Run("receive completed update result", func() { - for _, p := range []enumspb.WorkflowIdConflictPolicy{ - enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - } { - s.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func() { - tv := testvars.New(s.T()) - - startReq := startWorkflowReq(tv) - updReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - - // 1st update-with-start - uwsCh1 := sendUpdateWithStart(testcore.NewContext(), startReq, updReq) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes1 := <-uwsCh1 - s.NoError(uwsRes1.err) - - // 2nd update-with-start: using *same* UpdateID - but *different* RequestID - uwsRes2 := <-sendUpdateWithStart(testcore.NewContext(), startReq, updReq) - s.NoError(uwsRes2.err) - - s.Equal(uwsRes1.response.Responses[0].GetStartWorkflow().RunId, uwsRes2.response.Responses[0].GetStartWorkflow().RunId) - s.Equal(uwsRes1.response.Responses[1].GetUpdateWorkflow().Outcome.String(), uwsRes2.response.Responses[1].GetUpdateWorkflow().Outcome.String()) - - // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) - s.Equal(uwsRes1.response.Responses[1].GetUpdateWorkflow().Outcome.String(), pollRes.Outcome.String()) - }) - } - }) - - s.Run("dedupes start", func() { - for _, p := range []enumspb.WorkflowIdConflictPolicy{ - enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - } { - s.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func() { - tv := testvars.New(s.T()) - - startReq := startWorkflowReq(tv) - startReq.RequestId = "request_id" - startReq.WorkflowIdConflictPolicy = p - updReq1 := updateWorkflowRequest(s, tv.WithUpdateIDNumber(1), - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - - // 1st update-with-start - uwsCh1 := sendUpdateWithStart(testcore.NewContext(), startReq, updReq1) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes1 := <-uwsCh1 - s.NoError(uwsRes1.err) - - // 2nd update-with-start: using *same* RequestID - but *different* UpdateID - updReq2 := updateWorkflowRequest(s, tv.WithUpdateIDNumber(2), - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh2 := sendUpdateWithStart(testcore.NewContext(), startReq, updReq2) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes2 := <-uwsCh2 - s.NoError(uwsRes1.err) - - s.Equal(uwsRes1.response.Responses[0].GetStartWorkflow().RunId, uwsRes2.response.Responses[0].GetStartWorkflow().RunId) - }) - } - }) - }) - - s.Run("workflow is closed", func() { - - s.Run("workflow id reuse policy allow-duplicate", func() { - tv := testvars.New(s.T()) - - // start and terminate workflow - initialWorkflow, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), - &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - requireStartedAndRunning(s.T(), startResp) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) - - // ensure terminated workflow is not locked by update-with-start - err = s.SendSignal(s.Namespace().String(), &commonpb.WorkflowExecution{ - WorkflowId: tv.WorkflowID(), - RunId: initialWorkflow.RunId, - }, tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.ErrorContains(err, "workflow execution already completed") - - // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) - s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) - }) - - s.Run("workflow id reuse policy reject-duplicate", func() { - tv := testvars.New(s.T()) - - // start and terminate workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), - &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - uwsRes := <-uwsCh - s.Error(uwsRes.err) - s.Equal("Update-with-Start could not be executed.", uwsRes.err.Error()) - errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - s.Contains(errs[0].Error(), "Workflow execution already finished") - var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted - s.ErrorAs(errs[0], &alreadyStartedErr) - s.Equal("Operation was aborted.", errs[1].Error()) - }) - - s.Run("receive completed update result", func() { - for _, p := range []enumspb.WorkflowIdConflictPolicy{ - enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - } { - s.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func() { - tv := testvars.New(s.T()) - - // 1st update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = p - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - Commands: s.UpdateAcceptCompleteCommands(tv), - }, nil - }) - s.NoError(err) - - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - startResp1 := uwsRes.response.Responses[0].GetStartWorkflow() - _ = uwsRes.response.Responses[1].GetUpdateWorkflow() - requireStartedAndRunning(s.T(), startResp1) - - // terminate workflow - _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), - &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) - - // 2nd update-with-start (using the same Update ID but different Request ID) - uwsRes = <-sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - s.NoError(uwsRes.err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - s.False(startResp.Started) - s.Equal(startResp.Status, enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED) - // TODO: check startResp.Running - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) - }) - } - }) - }) - - s.Run("workflow start conflict", func() { - - s.Run("workflow id conflict policy fail: use-existing", func() { - tv := testvars.New(s.T()) - - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - - // simulate a race condition - s.InjectHook(testhooks.UpdateWithStartInBetweenLockAndStart, func() { - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startReq) + t.Run("update is aborted by closing workflow", func(t *testing.T) { + t.Run("retry request once when workflow was not started", func(t *testing.T) { + s := testcore.NewEnv(t) + // start workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) s.NoError(err) - }) - - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{}, nil - }) - s.NoError(err) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - - <-uwsCh - }) - }) - - s.Run("update is aborted by closing workflow", func() { - - s.Run("retry request once when workflow was not started", func() { - tv := testvars.New(s.T()) - - // start workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - // wait until the update is admitted - then complete workflow - waitUpdateAdmitted(s, tv) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, - }, - }, - }, - }, nil - }) - s.NoError(err) - - // update-with-start will do a server-side retry - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - }) - - s.Run("return retryable error after retry", func() { - tv := testvars.New(s.T()) - - // start workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - // wait until the update is admitted - waitUpdateAdmitted(s, tv) - - s.InjectHook(testhooks.UpdateWithStartOnClosingWorkflowRetry, func() { - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) s.NoError(err) - }) - // complete workflow (twice including retry) - for i := 0; i < 2; i++ { - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + // wait until the update is admitted - then complete workflow + waitUpdateAdmitted(s, s.Tv()) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { return &workflowservice.RespondWorkflowTaskCompletedRequest{ Commands: []*commandpb.Command{ @@ -5652,165 +5618,198 @@ func (s *UpdateWorkflowSuite) TestUpdateWithStart() { }, nil }) s.NoError(err) - } - // ensure update-with-start returns retryable error - uwsRes := <-uwsCh - s.Error(uwsRes.err) - errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - s.Equal("Operation was aborted.", errs[0].Error()) - s.ErrorContains(errs[1], update.AbortedByWorkflowClosingErr.Error()) - s.IsType(&serviceerror.Aborted{}, errs[1]) - }) + // update-with-start will do a server-side retry - s.Run("do not retry when workflow was started", func() { - tv := testvars.New(s.T()) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + }) - // wait until the update is admitted - then complete workflow - waitUpdateAdmitted(s, tv) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + t.Run("return retryable error after retry", func(t *testing.T) { + // Uses InjectHook which requires a dedicated cluster to avoid conflicts with other tests. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + // start workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + // wait until the update is admitted + waitUpdateAdmitted(s, s.Tv()) + + s.InjectHook(testhooks.UpdateWithStartOnClosingWorkflowRetry, func() { + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) + }) + + // complete workflow (twice including retry) + for i := 0; i < 2; i++ { + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }, + }, + }, nil + }) + s.NoError(err) + } + + // ensure update-with-start returns retryable error + uwsRes := <-uwsCh + s.Error(uwsRes.err) + errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + s.Equal("Operation was aborted.", errs[0].Error()) + s.ErrorContains(errs[1], update.AbortedByWorkflowClosingErr.Error()) + s.ErrorAs(errs[1], new(*serviceerror.Aborted)) + }) + + t.Run("do not retry when workflow was started", func(t *testing.T) { + s := testcore.NewEnv(t) + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + // wait until the update is admitted - then complete workflow + waitUpdateAdmitted(s, s.Tv()) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, }, }, - }, + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.Error(uwsRes.err) + errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + s.ErrorContains(errs[1], update.AbortedByWorkflowClosingErr.Error()) + }) + }) + + t.Run("return update rate limit error", func(t *testing.T) { + // lower maximum total number of updates for testing purposes + s := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdates, 1), + ) + + ctx := testcore.NewContext() + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + + // allows 1st + updateReq := updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(0), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, ctx, startReq, updateReq) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), }, nil }) s.NoError(err) - uwsRes := <-uwsCh - s.Error(uwsRes.err) - errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - s.ErrorContains(errs[1], update.AbortedByWorkflowClosingErr.Error()) + s.NoError(uwsRes.err) + + // denies 2nd + updateReq = updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(1), updateReq.WaitPolicy) + select { + case <-sendUpdateWithStart(s, ctx, startReq, updateReq): + err = (<-sendUpdateWithStart(s, ctx, startReq, updateReq)).err + s.Error(err) + errs := err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + s.Equal("Operation was aborted.", errs[0].Error()) + s.Contains(errs[1].Error(), "limit on the total number of distinct updates in this workflow has been reached") + case <-ctx.Done(): + s.Fail("timed out waiting for update") + } + }) + + t.Run("return update in-flight limit error", func(t *testing.T) { + // lower maximum in-flight updates for testing purposes + maxInFlight := 1 + s := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.WorkflowExecutionMaxInFlightUpdates, maxInFlight), + ) + + ctx := testcore.NewContext() + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + + // Start workflow and admit 1st update (but don't complete it) + updateReq := updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(0), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, ctx, startReq, updateReq) + + // Poll workflow task but only accept, don't complete the update + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + + // Try to send 2nd update-with-start while 1st is still in-flight (not completed) + updateReq = updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(1), updateReq.WaitPolicy) + uwsCh = sendUpdateWithStart(s, ctx, startReq, updateReq) + select { + case uwsRes := <-uwsCh: + err = uwsRes.err + s.Error(err) + + var multiOpsErr *serviceerror.MultiOperationExecution + s.ErrorAs(err, &multiOpsErr) + + errs := multiOpsErr.OperationErrors() + s.Len(errs, 2) + s.Equal("Operation was aborted.", errs[0].Error()) + s.Contains(errs[1].Error(), "limit on number of concurrent in-flight updates has been reached") + + // Verify ResourceExhausted error is accessible with all details preserved + var resExhausted *serviceerror.ResourceExhausted + s.ErrorAs(errs[1], &resExhausted) + s.Equal(enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, resExhausted.Cause) + s.Equal(enumspb.RESOURCE_EXHAUSTED_SCOPE_NAMESPACE, resExhausted.Scope) + s.Contains(resExhausted.Message, "limit on number of concurrent in-flight updates") + case <-ctx.Done(): + s.Fail("timed out waiting for update") + } }) }) - - s.Run("return update rate limit error", func() { - // lower maximum total number of updates for testing purposes - maxTotalUpdates := 1 - cleanup := s.OverrideDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdates, maxTotalUpdates) - defer cleanup() - - ctx := testcore.NewContext() - tv := testvars.New(s.T()) - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - - // allows 1st - updateReq := updateWorkflowRequest(s, tv.WithUpdateIDNumber(0), - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(ctx, startReq, updateReq) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - - // denies 2nd - updateReq = updateWorkflowRequest(s, tv.WithUpdateIDNumber(1), updateReq.WaitPolicy) - select { - case <-sendUpdateWithStart(ctx, startReq, updateReq): - err = (<-sendUpdateWithStart(ctx, startReq, updateReq)).err - s.Error(err) - errs := err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - s.Equal("Operation was aborted.", errs[0].Error()) - s.Contains(errs[1].Error(), "limit on the total number of distinct updates in this workflow has been reached") - case <-ctx.Done(): - s.Fail("timed out waiting for update") - } - }) - - s.Run("return update in-flight limit error", func() { - // lower maximum in-flight updates for testing purposes - maxInFlight := 1 - cleanup := s.OverrideDynamicConfig(dynamicconfig.WorkflowExecutionMaxInFlightUpdates, maxInFlight) - defer cleanup() - - ctx := testcore.NewContext() - tv := testvars.New(s.T()) - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - - // Start workflow and admit 1st update (but don't complete it) - updateReq := updateWorkflowRequest(s, tv.WithUpdateIDNumber(0), - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(ctx, startReq, updateReq) - - // Poll workflow task but only accept, don't complete the update - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - - // Try to send 2nd update-with-start while 1st is still in-flight (not completed) - updateReq = updateWorkflowRequest(s, tv.WithUpdateIDNumber(1), updateReq.WaitPolicy) - uwsCh = sendUpdateWithStart(ctx, startReq, updateReq) - select { - case uwsRes := <-uwsCh: - err = uwsRes.err - s.Error(err) - - var multiOpsErr *serviceerror.MultiOperationExecution - s.ErrorAs(err, &multiOpsErr) - - errs := multiOpsErr.OperationErrors() - s.Len(errs, 2) - s.Equal("Operation was aborted.", errs[0].Error()) - s.Contains(errs[1].Error(), "limit on number of concurrent in-flight updates has been reached") - - // Verify ResourceExhausted error is accessible with all details preserved - var resExhausted *serviceerror.ResourceExhausted - s.ErrorAs(errs[1], &resExhausted) - s.Equal(enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, resExhausted.Cause) - s.Equal(enumspb.RESOURCE_EXHAUSTED_SCOPE_NAMESPACE, resExhausted.Scope) - s.Contains(resExhausted.Message, "limit on number of concurrent in-flight updates") - case <-ctx.Done(): - s.Fail("timed out waiting for update") - } - }) -} - -func (s *UpdateWorkflowSuite) closeShard(wid string) { - s.T().Helper() - - resp, err := s.FrontendClient().DescribeNamespace(testcore.NewContext(), &workflowservice.DescribeNamespaceRequest{ - Namespace: s.Namespace().String(), - }) - s.NoError(err) - - _, err = s.AdminClient().CloseShard(testcore.NewContext(), &adminservice.CloseShardRequest{ - ShardId: common.WorkflowIDToHistoryShard(resp.NamespaceInfo.Id, wid, s.GetTestClusterConfig().HistoryConfig.NumHistoryShards), - }) - s.NoError(err) -} - -func (s *UpdateWorkflowSuite) useRunID(tv *testvars.TestVars, useRunID bool, runID string) *testvars.TestVars { - if useRunID { - return tv.WithRunID(runID) - } - return tv } diff --git a/tests/update_workflow_utils.go b/tests/update_workflow_utils.go index 2717461bf4..d2ac9fc54b 100644 --- a/tests/update_workflow_utils.go +++ b/tests/update_workflow_utils.go @@ -19,22 +19,22 @@ type updateResponseErr struct { err error } -func sendUpdate(ctx context.Context, s testEnv, tv *testvars.TestVars) <-chan updateResponseErr { +func sendUpdate(ctx context.Context, s testcore.Env, tv *testvars.TestVars) <-chan updateResponseErr { s.T().Helper() return sendUpdateInternal(ctx, s, tv, nil, false) } -func sendUpdateNoError(s testEnv, tv *testvars.TestVars) <-chan *workflowservice.UpdateWorkflowExecutionResponse { +func sendUpdateNoError(s testcore.Env, tv *testvars.TestVars) <-chan *workflowservice.UpdateWorkflowExecutionResponse { s.T().Helper() return sendUpdateNoErrorInternal(s, tv, nil) } -func sendUpdateNoErrorWaitPolicyAccepted(s testEnv, tv *testvars.TestVars) <-chan *workflowservice.UpdateWorkflowExecutionResponse { +func sendUpdateNoErrorWaitPolicyAccepted(s testcore.Env, tv *testvars.TestVars) <-chan *workflowservice.UpdateWorkflowExecutionResponse { s.T().Helper() return sendUpdateNoErrorInternal(s, tv, &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) } -func pollUpdate(s testEnv, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy) (*workflowservice.PollWorkflowExecutionUpdateResponse, error) { +func pollUpdate(s testcore.Env, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy) (*workflowservice.PollWorkflowExecutionUpdateResponse, error) { s.T().Helper() return s.FrontendClient().PollWorkflowExecutionUpdate(testcore.NewContext(), &workflowservice.PollWorkflowExecutionUpdateRequest{ Namespace: s.Namespace().String(), @@ -47,7 +47,7 @@ func pollUpdate(s testEnv, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolic } func updateWorkflowRequest( - s testEnv, + s testcore.Env, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy, ) *workflowservice.UpdateWorkflowExecutionRequest { @@ -65,7 +65,7 @@ func updateWorkflowRequest( } } -func sendUpdateNoErrorInternal(s testEnv, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy) <-chan *workflowservice.UpdateWorkflowExecutionResponse { +func sendUpdateNoErrorInternal(s testcore.Env, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy) <-chan *workflowservice.UpdateWorkflowExecutionResponse { s.T().Helper() retCh := make(chan *workflowservice.UpdateWorkflowExecutionResponse) syncCh := make(chan struct{}) @@ -80,7 +80,7 @@ func sendUpdateNoErrorInternal(s testEnv, tv *testvars.TestVars, waitPolicy *upd func sendUpdateInternal( ctx context.Context, - s testEnv, + s testcore.Env, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy, requireNoError bool, @@ -99,7 +99,7 @@ func sendUpdateInternal( return updateResultCh } -func waitUpdateAdmitted(s testEnv, tv *testvars.TestVars) { +func waitUpdateAdmitted(s testcore.Env, tv *testvars.TestVars) { s.T().Helper() require.EventuallyWithTf(s.T(), func(collect *assert.CollectT) { pollResp, pollErr := s.FrontendClient().PollWorkflowExecutionUpdate(testcore.NewContext(), &workflowservice.PollWorkflowExecutionUpdateRequest{ diff --git a/tests/workflow_utils.go b/tests/workflow_utils.go index ca84a71aec..2f8f351540 100644 --- a/tests/workflow_utils.go +++ b/tests/workflow_utils.go @@ -1,21 +1,12 @@ package tests import ( - "testing" - "go.temporal.io/api/workflowservice/v1" - "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/tests/testcore" ) -type testEnv interface { - T() *testing.T - Namespace() namespace.Name - FrontendClient() workflowservice.WorkflowServiceClient -} - -func mustStartWorkflow(s testEnv, tv *testvars.TestVars) string { +func mustStartWorkflow(s testcore.Env, tv *testvars.TestVars) string { s.T().Helper() startResp, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowRequest(s, tv)) if err != nil { @@ -24,7 +15,7 @@ func mustStartWorkflow(s testEnv, tv *testvars.TestVars) string { return startResp.GetRunId() } -func startWorkflowRequest(s testEnv, tv *testvars.TestVars) *workflowservice.StartWorkflowExecutionRequest { +func startWorkflowRequest(s testcore.Env, tv *testvars.TestVars) *workflowservice.StartWorkflowExecutionRequest { return &workflowservice.StartWorkflowExecutionRequest{ RequestId: tv.Any().String(), Namespace: s.Namespace().String(), diff --git a/tests/xdc/base.go b/tests/xdc/base.go index f0568fa9ec..c5a0846080 100644 --- a/tests/xdc/base.go +++ b/tests/xdc/base.go @@ -101,6 +101,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { s.dynamicConfigOverrides[dynamicconfig.VisibilityProcessorMaxPollInterval.Key()] = time.Second * 3 s.dynamicConfigOverrides[dynamicconfig.OutboundProcessorMaxPollInterval.Key()] = time.Second * 3 + persistenceDefaults := testcore.GetPersistenceTestDefaults() clusterConfigs := []*testcore.TestClusterConfig{ { ClusterMetadata: cluster.Config{ @@ -110,6 +111,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { HistoryConfig: testcore.HistoryConfig{ NumHistoryShards: cmp.Or(params.NumHistoryShards, 1), }, + Persistence: persistenceDefaults, }, { ClusterMetadata: cluster.Config{ @@ -119,6 +121,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { HistoryConfig: testcore.HistoryConfig{ NumHistoryShards: cmp.Or(params.NumHistoryShards, 1), }, + Persistence: persistenceDefaults, }, } @@ -131,7 +134,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { clusterConfigs[clusterIndex].ClusterMetadata.MasterClusterName = clusterName clusterConfigs[clusterIndex].ClusterMetadata.CurrentClusterName = clusterName clusterConfigs[clusterIndex].ClusterMetadata.EnableGlobalNamespace = true - clusterConfigs[clusterIndex].Persistence.DBName = "func_tests_" + clusterName + clusterConfigs[clusterIndex].Persistence.DBName += "_" + clusterName clusterConfigs[clusterIndex].ClusterMetadata.ClusterInformation = map[string]cluster.ClusterInformation{ clusterName: { Enabled: true, diff --git a/tests/xdc/replication_enable_test.go b/tests/xdc/replication_enable_test.go index 4f807ef7ee..4aea135f82 100644 --- a/tests/xdc/replication_enable_test.go +++ b/tests/xdc/replication_enable_test.go @@ -51,6 +51,7 @@ func (s *ReplicationEnableTestSuite) SetupSuite() { dynamicconfig.SendRawHistoryBetweenInternalServices.Key(): true, } + persistenceDefaults := testcore.GetPersistenceTestDefaults() clusterConfigs := []*testcore.TestClusterConfig{ { ClusterMetadata: cluster.Config{ @@ -60,6 +61,7 @@ func (s *ReplicationEnableTestSuite) SetupSuite() { HistoryConfig: testcore.HistoryConfig{ NumHistoryShards: 1, }, + Persistence: persistenceDefaults, }, { ClusterMetadata: cluster.Config{ @@ -69,6 +71,7 @@ func (s *ReplicationEnableTestSuite) SetupSuite() { HistoryConfig: testcore.HistoryConfig{ NumHistoryShards: 1, }, + Persistence: persistenceDefaults, }, } @@ -82,7 +85,7 @@ func (s *ReplicationEnableTestSuite) SetupSuite() { clusterConfigs[clusterIndex].ClusterMetadata.MasterClusterName = clusterName clusterConfigs[clusterIndex].ClusterMetadata.CurrentClusterName = clusterName clusterConfigs[clusterIndex].ClusterMetadata.EnableGlobalNamespace = true - clusterConfigs[clusterIndex].Persistence.DBName = "func_tests_" + clusterName + clusterConfigs[clusterIndex].Persistence.DBName += "_" + clusterName clusterConfigs[clusterIndex].ClusterMetadata.ClusterInformation = map[string]cluster.ClusterInformation{ clusterName: { Enabled: true,