From f5bfe42239d23fc8791d59e91ebadbee2837a0e2 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Fri, 23 Jan 2026 20:08:49 -0800 Subject: [PATCH] Parallel Workflow Update Tests (#8811) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? Migrated `TestWorkflowUpdateSuite` away from testify's `Suite`; enabling parallel test execution. **How it works** - a test invokes `testcore.NewEnv(t)` to obtain a new `TestEnv` - `TestEnv` sets `t.Parallel()` (_intentionally not giving a way to opt out!_) - `TestEnv` obtains a test cluster from `clusterPool` (_or blocks if all are in-use right now_) - env var `TEMPORAL_TEST_SHARED_CLUSTERS` controls size of the pool - if a test relies on APIs like InjectHook, a dedicated cluster is used to prevent overlap - env var `TEMPORAL_TEST_DEDICATED_CLUSTERS` controls number of dedicated clusters **testify suites** Existing test suites are limited by the same dedicated cluster pool to prevent creating too many clusters. **Database connections** SQLite setup for TestEnv-based func tests (ie only TestWorkflowUpdateSuite so far) has been changed to a file-based approach since that supports much better concurrency due to its WAL that an in-memory SQLite database does not support. Connection limits for other databases were also raised due to connection errors. **Planned follow-ups** - Migrating the other testify suites should be fairly straight-forward with the use of AI agents. - Reduce need for dedicated clusters by leveraging isolated namespace-per-test more. - Eliminate all `time.Sleep`s. - Tweak test cluster pool behavior. ## Why? 1. **Local speedup**: benchmarks show a ~50% speed increase (36.1s → 16.6s) for `TestWorkflowUpdateSuite`. 5. **Namespace isolation**: every test runs in its own namespace. This greatly reduces the risk of (accidental) collisions and also reduces the need to craft unique identifiers such as for task queues and workflow IDs. 6. **Deprecate testify suites**: Long-term strategy to remove use of testify suites in functional tests (one reason being their inability to run tests within a suite in parallel). ## How did you test it? - [ ] built - [ ] run locally and tested manually - [x] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) ## Potential Issues 1. Logs become less useful since there is more interleaving of tests. 2. Higher resource consumption: it requires more concurrent connections to databases and shows higher memory consumption (see 3 and 4). This could cause some short-term instability on CI. Note that some other PRs were merged to add mechanics for monitoring memory usage much better; which will help here. 4. Until all functional tests are converted, there is an imbalance in test cluster creation: migrated tests use the shared pool while current tests create one cluster each. Especially given the fact that some tests don't allow for test cluster sharing as they use non-parallelizable actions such as `InjectHook` or dynamic config overrides. With some more effort the number of these can be reduced. 7. Setup of test clusters was designed around the idea of short-lived clusters, one per suite. But when re-using them for longer, some of the assumptions don't hold anymore and increase memory usage. There's a band aid in place to limit how often a test cluster can be used before it's torn down. A long-term solution requires some design changes to how test clusters are started/used/torn down. 8. If there are certain cross-namespace issues or bugs that affect multiple tests, it might be harder to identify the root cause now. However; the existing test re-runs should at least mitigate these short-term. --------- Co-authored-by: Dan Davison --- .github/workflows/run-tests.yml | 39 - .../persistence_test_base.go | 21 +- common/persistence/persistence-tests/setup.go | 66 +- .../persistence/sql/test_sql_persistence.go | 9 +- develop/github/docker-compose.yml | 4 + docs/development/testing.md | 3 +- tests/testcore/functional_test_base.go | 45 +- tests/testcore/test_cluster.go | 41 +- tests/testcore/test_cluster_pool.go | 189 + tests/testcore/test_env.go | 188 + tests/update_workflow_test.go | 8963 ++++++++--------- tests/update_workflow_utils.go | 16 +- tests/workflow_utils.go | 13 +- tests/xdc/base.go | 5 +- tests/xdc/replication_enable_test.go | 5 +- 15 files changed, 5013 insertions(+), 4594 deletions(-) create mode 100644 tests/testcore/test_cluster_pool.go create mode 100644 tests/testcore/test_env.go diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index db0da0a964..0a9630df8d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -534,8 +534,6 @@ jobs: run: ./develop/github/monitor_test.sh make functional-test-coverage env: TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }} - TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput - TEMPORAL_OTEL_DEBUG: true - name: Print memory snapshot if: always() @@ -595,17 +593,6 @@ jobs: FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_test_suites }} -count=5" - # Upload OpenTelemetry traces. - - name: Upload OpenTelemetry traces - if: always() - uses: actions/upload-artifact@v4 - with: - name: traces-json--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{matrix.shard_index}}--${{github.run_attempt}}--${{matrix.name}}--functional-test - path: ./.testoutput/traces.*.json - if-no-files-found: ignore - include-hidden-files: true - retention-days: 28 - # XDC matrix job. Include `${{ matrix.name }}` in the display name so each # matrix variant has a unique name for the job-id lookup. functional-test-xdc: @@ -708,9 +695,6 @@ jobs: - name: Run functional test xdc timeout-minutes: 35 # update this to TEST_TIMEOUT+5 if you update the Makefile run: ./develop/github/monitor_test.sh make functional-test-xdc-coverage - env: - TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput - TEMPORAL_OTEL_DEBUG: true - name: Print memory snapshot if: always() @@ -770,16 +754,6 @@ jobs: FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_xdc_test_suites }} -count=5" - # Upload OpenTelemetry traces. - - name: Upload OpenTelemetry traces - if: always() - uses: actions/upload-artifact@v4 - with: - name: traces-json--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{github.run_attempt}}--${{matrix.name}}--functional-test-xdc - path: ./.testoutput/traces.*.json - if-no-files-found: ignore - retention-days: 28 - # NDC matrix job. Include `${{ matrix.name }}` in the display name so each # matrix variant has a unique name for the job-id lookup. functional-test-ndc: @@ -879,9 +853,6 @@ jobs: - name: Run functional test ndc timeout-minutes: 15 run: ./develop/github/monitor_test.sh make functional-test-ndc-coverage - env: - TEMPORAL_TEST_OTEL_OUTPUT: ${{ github.workspace }}/.testoutput - TEMPORAL_OTEL_DEBUG: true - name: Print memory snapshot if: always() @@ -931,16 +902,6 @@ jobs: FAILED_TEST_RETRIES: "0" # not retrying failed tests intentionally here since we're trying to detect flakes TEST_ARGS: "-run=${{ needs.set-up-single-test.outputs.modified_functional_ndc_test_suites }} -count=5" - # Upload OpenTelemetry traces. - - name: Upload OpenTelemetry traces - if: always() - uses: actions/upload-artifact@v4 - with: - name: traces-json--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{github.run_attempt}}--${{matrix.name}}--functional-test-ndc - path: ./.testoutput/traces.*.json - if-no-files-found: ignore - retention-days: 28 - test-status: if: always() name: Test Status diff --git a/common/persistence/persistence-tests/persistence_test_base.go b/common/persistence/persistence-tests/persistence_test_base.go index e0699e490c..e53079a3eb 100644 --- a/common/persistence/persistence-tests/persistence_test_base.go +++ b/common/persistence/persistence-tests/persistence_test_base.go @@ -1,6 +1,7 @@ package persistencetests import ( + "cmp" "context" "fmt" "math/rand" @@ -61,7 +62,24 @@ type ( FaultInjection *config.FaultInjection Logger log.Logger `yaml:"-"` } +) +// ApplyDefaults copies database configuration from src, preserving any non-zero values already set. +func (o *TestBaseOptions) ApplyDefaults(src *TestBaseOptions) { + o.StoreType = cmp.Or(o.StoreType, src.StoreType) + o.SQLDBPluginName = cmp.Or(o.SQLDBPluginName, src.SQLDBPluginName) + o.DBName = cmp.Or(o.DBName, src.DBName) + o.DBUsername = cmp.Or(o.DBUsername, src.DBUsername) + o.DBPassword = cmp.Or(o.DBPassword, src.DBPassword) + o.DBHost = cmp.Or(o.DBHost, src.DBHost) + o.DBPort = cmp.Or(o.DBPort, src.DBPort) + o.SchemaDir = cmp.Or(o.SchemaDir, src.SchemaDir) + if o.ConnectAttributes == nil { + o.ConnectAttributes = src.ConnectAttributes + } +} + +type ( // TestBase wraps the base setup needed to create workflows over persistence layer. TestBase struct { suite.Suite @@ -119,9 +137,6 @@ func NewTestClusterForCassandra(options *TestBaseOptions, logger log.Logger) *ca // NewTestBaseWithSQL returns a new persistence test base backed by SQL func NewTestBaseWithSQL(options *TestBaseOptions) *TestBase { - if options.DBName == "" { - options.DBName = "test_" + GenerateRandomDBName(3) - } logger := options.Logger if logger == nil { logger = log.NewTestLogger() diff --git a/common/persistence/persistence-tests/setup.go b/common/persistence/persistence-tests/setup.go index 6f16480ad5..1dd765ba6c 100644 --- a/common/persistence/persistence-tests/setup.go +++ b/common/persistence/persistence-tests/setup.go @@ -1,6 +1,10 @@ package persistencetests import ( + "fmt" + "os" + "path/filepath" + "go.temporal.io/server/common/config" "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql" "go.temporal.io/server/common/persistence/sql/sqlplugin/postgresql" @@ -9,6 +13,8 @@ import ( ) const ( + testCassandraSchemaDir = "schema/cassandra/" + testMySQLUser = "temporal" testMySQLPassword = "temporal" testMySQLSchemaDir = "schema/mysql/v8" @@ -24,10 +30,45 @@ const ( testSQLiteSchemaDir = "schema/sqlite/v3" // specify if mode is not "memory" ) +// GetTestClusterOption returns test options for the given store type and driver. +func GetTestClusterOption(storeType, driver string) *TestBaseOptions { + switch storeType { + case config.StoreTypeSQL: + switch driver { + case mysql.PluginName: + return GetMySQLTestClusterOption() + case postgresql.PluginName: + return GetPostgreSQLTestClusterOption() + case postgresql.PluginNamePGX: + return GetPostgreSQLPGXTestClusterOption() + case sqlite.PluginName: + return GetSQLiteMemoryTestClusterOption() + default: + panic(fmt.Sprintf("unknown sql driver: %v", driver)) + } + case config.StoreTypeNoSQL: + return GetCassandraTestClusterOption() + default: + panic(fmt.Sprintf("unknown store type: %v", storeType)) + } +} + +// GetCassandraTestClusterOption returns test options +func GetCassandraTestClusterOption() *TestBaseOptions { + return &TestBaseOptions{ + DBName: "test_" + GenerateRandomDBName(3), + DBHost: environment.GetCassandraAddress(), + DBPort: environment.GetCassandraPort(), + SchemaDir: testCassandraSchemaDir, + StoreType: config.StoreTypeNoSQL, + } +} + // GetMySQLTestClusterOption return test options func GetMySQLTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ SQLDBPluginName: mysql.PluginName, + DBName: "test_" + GenerateRandomDBName(3), DBUsername: testMySQLUser, DBPassword: testMySQLPassword, DBHost: environment.GetMySQLAddress(), @@ -41,6 +82,7 @@ func GetMySQLTestClusterOption() *TestBaseOptions { func GetPostgreSQLTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ SQLDBPluginName: postgresql.PluginName, + DBName: "test_" + GenerateRandomDBName(3), DBUsername: testPostgreSQLUser, DBPassword: testPostgreSQLPassword, DBHost: environment.GetPostgreSQLAddress(), @@ -54,6 +96,7 @@ func GetPostgreSQLTestClusterOption() *TestBaseOptions { func GetPostgreSQLPGXTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ SQLDBPluginName: postgresql.PluginNamePGX, + DBName: "test_" + GenerateRandomDBName(3), DBUsername: testPostgreSQLUser, DBPassword: testPostgreSQLPassword, DBHost: environment.GetPostgreSQLAddress(), @@ -66,14 +109,20 @@ func GetPostgreSQLPGXTestClusterOption() *TestBaseOptions { // GetSQLiteFileTestClusterOption return test options func GetSQLiteFileTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ - SQLDBPluginName: sqlite.PluginName, - DBUsername: testSQLiteUser, - DBPassword: testSQLitePassword, - DBHost: environment.GetLocalhostIP(), - DBPort: 0, - SchemaDir: testSQLiteSchemaDir, - StoreType: config.StoreTypeSQL, - ConnectAttributes: map[string]string{"cache": testSQLiteCache}, + SQLDBPluginName: sqlite.PluginName, + DBName: filepath.Join(os.TempDir(), "test_"+GenerateRandomDBName(3)), // put files in temp to avoid cluttering the project + DBUsername: testSQLiteUser, + DBPassword: testSQLitePassword, + DBHost: environment.GetLocalhostIP(), + DBPort: 0, + SchemaDir: testSQLiteSchemaDir, + StoreType: config.StoreTypeSQL, + ConnectAttributes: map[string]string{ + "cache": "shared", + "busy_timeout": "30000", + "journal_mode": "wal", + "synchronous": "normal", + }, } } @@ -81,6 +130,7 @@ func GetSQLiteFileTestClusterOption() *TestBaseOptions { func GetSQLiteMemoryTestClusterOption() *TestBaseOptions { return &TestBaseOptions{ SQLDBPluginName: sqlite.PluginName, + DBName: "test_" + GenerateRandomDBName(3), DBUsername: testSQLiteUser, DBPassword: testSQLitePassword, DBHost: environment.GetLocalhostIP(), diff --git a/common/persistence/sql/test_sql_persistence.go b/common/persistence/sql/test_sql_persistence.go index 5a48793ba8..823dbd170f 100644 --- a/common/persistence/sql/test_sql_persistence.go +++ b/common/persistence/sql/test_sql_persistence.go @@ -143,10 +143,11 @@ func (s *TestCluster) DropDatabase() { if cfg2.PluginName == "sqlite" && cfg2.DatabaseName != ":memory:" && cfg2.ConnectAttributes["mode"] != "memory" { if len(cfg2.DatabaseName) > 3 { // 3 should mean not ., .., empty, or / - err := os.Remove(cfg2.DatabaseName) - if err != nil { - panic(err) - } + // Remove main database file + _ = os.Remove(cfg2.DatabaseName) + // Remove WAL mode files (may not exist if WAL wasn't used) + _ = os.Remove(cfg2.DatabaseName + "-wal") + _ = os.Remove(cfg2.DatabaseName + "-shm") } return } diff --git a/develop/github/docker-compose.yml b/develop/github/docker-compose.yml index 6e8dfa9ebe..add741ff62 100644 --- a/develop/github/docker-compose.yml +++ b/develop/github/docker-compose.yml @@ -7,6 +7,8 @@ services: CASSANDRA_LISTEN_ADDRESS: 127.0.0.1 MAX_HEAP_SIZE: "2G" HEAP_NEWSIZE: "200M" + # Increase native transport threads for handling more concurrent connections + JVM_EXTRA_OPTS: "-Dcassandra.native_transport_max_threads=512" mysql: image: mysql:8.0.29-oracle @@ -14,6 +16,7 @@ services: - "3306:3306" environment: MYSQL_ROOT_PASSWORD: root + command: --max-connections=500 volumes: - ./mysql-init:/docker-entrypoint-initdb.d @@ -24,6 +27,7 @@ services: environment: POSTGRES_USER: temporal POSTGRES_PASSWORD: temporal + command: postgres -c max_connections=500 volumes: - ./postgresql-init:/docker-entrypoint-initdb.d diff --git a/docs/development/testing.md b/docs/development/testing.md index 72b3d05780..195dbcdf2c 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -14,7 +14,8 @@ This document describes the project's testing setup, utilities and best practice - `TEMPORAL_TEST_LOG_FORMAT`: Controls the output format for test logs. Available options: `json` or `console` - `TEMPORAL_TEST_LOG_LEVEL`: Sets the verbosity level for test logging. Available levels: `debug`, `info`, `warn`, `error`, `fatal` - `TEMPORAL_TEST_OTEL_OUTPUT`: Enables OpenTelemetry (OTEL) trace output for failed tests to the provided file path. - +- `TEMPORAL_TEST_SHARED_CLUSTERS`: Number of shared clusters in the pool. Each can be used by multiple tests simultaneously. +- `TEMPORAL_TEST_DEDICATED_CLUSTERS`: Number of dedicated clusters in the pool. Each can be used by one test only at a time. ### Debugging via IDE #### GoLand diff --git a/tests/testcore/functional_test_base.go b/tests/testcore/functional_test_base.go index 1dbc580191..2106c293df 100644 --- a/tests/testcore/functional_test_base.go +++ b/tests/testcore/functional_test_base.go @@ -35,6 +35,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/persistence" + persistencetests "go.temporal.io/server/common/persistence/persistence-tests" "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/rpc" @@ -85,6 +86,11 @@ type ( // TODO (alex): replace with v2 taskPoller *taskpoller.TaskPoller + + // isShared indicates whether this cluster is shared between multiple tests. + // Certain operations (e.g. InjectHook, CloseShard) are not safe on shared clusters + // and will panic if called. + isShared bool } // TestClusterParams contains the variables which are used to configure test cluster via the TestClusterOption type. TestClusterParams struct { @@ -94,6 +100,7 @@ type ( EnableMTLS bool FaultInjectionConfig *config.FaultInjection NumHistoryShards int32 + SharedCluster bool } TestClusterOption func(params *TestClusterParams) ) @@ -156,6 +163,12 @@ func WithNumHistoryShards(n int32) TestClusterOption { } } +func WithSharedCluster() TestClusterOption { + return func(params *TestClusterParams) { + params.SharedCluster = true + } +} + func (s *FunctionalTestBase) GetTestCluster() *TestCluster { return s.testCluster } @@ -228,6 +241,12 @@ func (s *FunctionalTestBase) TearDownSuite() { } func (s *FunctionalTestBase) SetupSuiteWithCluster(options ...TestClusterOption) { + // Acquire a slot from the dedicated test cluster pool. + testClusterPool.dedicated.acquireSlot(s.T()) + s.setupCluster(options...) +} + +func (s *FunctionalTestBase) setupCluster(options ...TestClusterOption) { params := ApplyTestClusterOptions(options) // NOTE: A suite might set its own logger. Example: AcquireShardSuiteBase. @@ -254,6 +273,13 @@ func (s *FunctionalTestBase) SetupSuiteWithCluster(options ...TestClusterOption) EnableMTLS: params.EnableMTLS, } + // Apply configuration for shared clusters. + if params.SharedCluster { + // Use file-based SQLite for shared clusters to support parallel test access. + s.testClusterConfig.Persistence = *persistencetests.GetSQLiteFileTestClusterOption() + s.isShared = true + } + // Initialize the OTEL collector if OTEL is enabled. // Must be done before the test cluster is created, so that the collector can be used by the test cluster. if otelOutputDir := os.Getenv("TEMPORAL_TEST_OTEL_OUTPUT"); otelOutputDir != "" { @@ -568,17 +594,30 @@ func (s *FunctionalTestBase) DurationNear(value, target, tolerance time.Duration s.Less(value, target+tolerance) } -// Overrides one dynamic config setting for the duration of this test (or sub-test). The change -// will automatically be reverted at the end of the test (using t.Cleanup). The cleanup -// function is also returned if you want to revert the change before the end of the test. func (s *FunctionalTestBase) OverrideDynamicConfig(setting dynamicconfig.GenericSetting, value any) (cleanup func()) { return s.testCluster.host.overrideDynamicConfig(s.T(), setting.Key(), value) } func (s *FunctionalTestBase) InjectHook(key testhooks.Key, value any) (cleanup func()) { + if s.isShared { + s.T().Fatalf("InjectHook cannot be called on a shared cluster; use testcore.WithDedicatedCluster()") + } return s.testCluster.host.injectHook(s.T(), key, value) } +// CloseShard closes the shard that contains the given workflow. +// This is a cluster-global operation and cannot be called on shared clusters. +func (s *FunctionalTestBase) CloseShard(namespaceID string, workflowID string) { + if s.isShared { + s.T().Fatalf("CloseShard cannot be called on a shared cluster; use testcore.WithDedicatedCluster()") + } + shardID := common.WorkflowIDToHistoryShard(namespaceID, workflowID, s.testClusterConfig.HistoryConfig.NumHistoryShards) + _, err := s.AdminClient().CloseShard(NewContext(), &adminservice.CloseShardRequest{ + ShardId: shardID, + }) + s.Require().NoError(err) +} + func (s *FunctionalTestBase) GetNamespaceID(namespace string) string { namespaceResp, err := s.FrontendClient().DescribeNamespace(NewContext(), &workflowservice.DescribeNamespaceRequest{ Namespace: namespace, diff --git a/tests/testcore/test_cluster.go b/tests/testcore/test_cluster.go index 41eb5b8d7d..d8eeef4f73 100644 --- a/tests/testcore/test_cluster.go +++ b/tests/testcore/test_cluster.go @@ -35,9 +35,6 @@ import ( "go.temporal.io/server/common/namespace/nsreplication" "go.temporal.io/server/common/persistence" persistencetests "go.temporal.io/server/common/persistence/persistence-tests" - "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql" - "go.temporal.io/server/common/persistence/sql/sqlplugin/postgresql" - "go.temporal.io/server/common/persistence/sql/sqlplugin/sqlite" esclient "go.temporal.io/server/common/persistence/visibility/store/elasticsearch/client" "go.temporal.io/server/common/pprof" "go.temporal.io/server/common/primitives" @@ -144,37 +141,15 @@ type PersistenceTestBaseFactory interface { type defaultPersistenceTestBaseFactory struct{} +// GetPersistenceTestDefaults returns the default persistence options based on CLI flags. +// Use this when creating TestClusterConfig to ensure proper database configuration. +func GetPersistenceTestDefaults() persistencetests.TestBaseOptions { + return *persistencetests.GetTestClusterOption(cliFlags.persistenceType, cliFlags.persistenceDriver) +} + func (f *defaultPersistenceTestBaseFactory) NewTestBase(options *persistencetests.TestBaseOptions) *persistencetests.TestBase { - options.StoreType = cliFlags.persistenceType - switch cliFlags.persistenceType { - case config.StoreTypeSQL: - var ops *persistencetests.TestBaseOptions - switch cliFlags.persistenceDriver { - case mysql.PluginName: - ops = persistencetests.GetMySQLTestClusterOption() - case postgresql.PluginName: - ops = persistencetests.GetPostgreSQLTestClusterOption() - case postgresql.PluginNamePGX: - ops = persistencetests.GetPostgreSQLPGXTestClusterOption() - case sqlite.PluginName: - ops = persistencetests.GetSQLiteMemoryTestClusterOption() - default: - //nolint:forbidigo // test code - panic(fmt.Sprintf("unknown sql store driver: %v", cliFlags.persistenceDriver)) - } - options.SQLDBPluginName = cliFlags.persistenceDriver - options.DBUsername = ops.DBUsername - options.DBPassword = ops.DBPassword - options.DBHost = ops.DBHost - options.DBPort = ops.DBPort - options.SchemaDir = ops.SchemaDir - options.ConnectAttributes = ops.ConnectAttributes - case config.StoreTypeNoSQL: - // noop for now - default: - //nolint:forbidigo // test code - panic(fmt.Sprintf("unknown store type: %v", options.StoreType)) - } + defaults := GetPersistenceTestDefaults() + options.ApplyDefaults(&defaults) if cliFlags.enableFaultInjection != "" && options.FaultInjection == nil { // If -enableFaultInjection is passed to the test runner, then default fault injection config is added to the persistence options. diff --git a/tests/testcore/test_cluster_pool.go b/tests/testcore/test_cluster_pool.go new file mode 100644 index 0000000000..28fa0dd846 --- /dev/null +++ b/tests/testcore/test_cluster_pool.go @@ -0,0 +1,189 @@ +package testcore + +import ( + "os" + "runtime" + "strconv" + "sync" + "sync/atomic" + "testing" + + "go.temporal.io/server/common/dynamicconfig" +) + +var testClusterPool *clusterPool + +func init() { + sharedSize := max(1, runtime.GOMAXPROCS(0)/2) + if v := os.Getenv("TEMPORAL_TEST_SHARED_CLUSTERS"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + panic("TEMPORAL_TEST_SHARED_CLUSTERS must be a positive integer") + } + sharedSize = n + } + + dedicatedSize := runtime.GOMAXPROCS(0) + if v := os.Getenv("TEMPORAL_TEST_DEDICATED_CLUSTERS"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + panic("TEMPORAL_TEST_DEDICATED_CLUSTERS must be a positive integer") + } + dedicatedSize = n + } + + // In CI, recreate clusters after 50 tests to prevent resource accumulation. + // Locally, clusters are reused indefinitely for faster iteration. + var maxUsage int + if os.Getenv("CI") != "" { + maxUsage = 50 + } + + sharedPool := newPool(sharedSize, false) + sharedPool.maxUsage = maxUsage + + dedicatedPool := newPool(dedicatedSize, true) + dedicatedPool.maxUsage = maxUsage + + testClusterPool = &clusterPool{ + shared: sharedPool, + dedicated: dedicatedPool, + } +} + +// pool manages a fixed number of test clusters with lazy initialization. +type pool struct { + clusters []*FunctionalTestBase + inits []sync.Once + counter atomic.Int64 // for round-robin (when slots is nil) + slots chan int // for exclusive access (nil means shared/concurrent access) + + // For shared pools: track usage and support teardown/recreate after maxUsage tests + usageCounts []atomic.Int64 + clusterMu []sync.Mutex // protects cluster teardown/recreate + maxUsage int // max tests per cluster before recreate (0 = unlimited) + createFn func() *FunctionalTestBase +} + +func newPool(size int, exclusive bool) *pool { + p := &pool{ + clusters: make([]*FunctionalTestBase, size), + inits: make([]sync.Once, size), + usageCounts: make([]atomic.Int64, size), + clusterMu: make([]sync.Mutex, size), + } + if exclusive { + p.slots = make(chan int, size) + for i := range size { + p.slots <- i + } + } + return p +} + +// get returns a cluster from the pool, creating it lazily if needed. +// For exclusive pools, blocks until a slot is available and registers cleanup. +// For shared pools, uses round-robin. +// Both pool types may recreate clusters after maxUsage tests (in CI). +func (p *pool) get(t *testing.T, createCluster func() *FunctionalTestBase) *FunctionalTestBase { + var idx int + if p.slots != nil { + idx = <-p.slots + t.Cleanup(func() { p.slots <- idx }) + } else { + idx = int(p.counter.Add(1)-1) % len(p.clusters) + } + + // Check if we need to recreate the cluster after maxUsage tests + if p.maxUsage > 0 { + usage := p.usageCounts[idx].Add(1) + if usage > int64(p.maxUsage) { + p.clusterMu[idx].Lock() + // Double-check after acquiring lock + if p.usageCounts[idx].Load() > int64(p.maxUsage) && p.clusters[idx] != nil { + if err := p.clusters[idx].testCluster.TearDownCluster(); err != nil { + t.Logf("Failed to tear down cluster %d: %v", idx, err) + } + p.clusters[idx] = createCluster() + p.usageCounts[idx].Store(1) // Reset to 1 (this test counts) + } + p.clusterMu[idx].Unlock() + } + } + + // Lazy initialization for first use + p.inits[idx].Do(func() { + p.clusters[idx] = createCluster() + }) + + cluster := p.clusters[idx] + cluster.SetT(t) + return cluster +} + +// acquireSlot gets exclusive access to a slot without using a pooled cluster. +// Used when a fresh cluster is needed (e.g., custom dynamic config). +func (p *pool) acquireSlot(t *testing.T) { + if p.slots == nil { + return + } + idx := <-p.slots + t.Cleanup(func() { p.slots <- idx }) +} + +type clusterPool struct { + shared *pool + dedicated *pool +} + +func (p *clusterPool) get(t *testing.T, dedicated bool, dynamicConfig map[dynamicconfig.Key]any) *FunctionalTestBase { + if dedicated || len(dynamicConfig) > 0 { + return p.getDedicated(t, dynamicConfig) + } + return p.getShared(t) +} + +func (p *clusterPool) getShared(t *testing.T) *FunctionalTestBase { + return p.shared.get(t, func() *FunctionalTestBase { + return p.createCluster(t, nil, true) + }) +} + +func (p *clusterPool) getDedicated(t *testing.T, dynamicConfig map[dynamicconfig.Key]any) *FunctionalTestBase { + if len(dynamicConfig) > 0 { + // Custom dynamic config requires a fresh cluster (can't reuse). + p.dedicated.acquireSlot(t) + cluster := p.createCluster(t, dynamicConfig, false) + + // Register cleanup to tear down the cluster when the test completes. + t.Cleanup(func() { + if err := cluster.testCluster.TearDownCluster(); err != nil { + t.Logf("Failed to tear down cluster: %v", err) + } + }) + + return cluster + } + + // If no custom dynamic config is provided, reuse an existing cluster. + return p.dedicated.get(t, func() *FunctionalTestBase { + return p.createCluster(t, nil, false) + }) +} + +func (p *clusterPool) createCluster(t *testing.T, dynamicConfig map[dynamicconfig.Key]any, shared bool) *FunctionalTestBase { + tbase := &FunctionalTestBase{} + tbase.SetT(t) + + var opts []TestClusterOption + if shared { + opts = append(opts, WithSharedCluster()) + } + if len(dynamicConfig) > 0 { + opts = append(opts, WithDynamicConfigOverrides(dynamicConfig)) + } + + tbase.setupCluster(opts...) + + return tbase +} diff --git a/tests/testcore/test_env.go b/tests/testcore/test_env.go new file mode 100644 index 0000000000..69eb7d1a00 --- /dev/null +++ b/tests/testcore/test_env.go @@ -0,0 +1,188 @@ +package testcore + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/historyrequire" + "go.temporal.io/server/common/testing/taskpoller" + "go.temporal.io/server/common/testing/testvars" +) + +var _ Env = (*testEnv)(nil) + +type Env interface { + T() *testing.T + Namespace() namespace.Name + FrontendClient() workflowservice.WorkflowServiceClient + GetTestCluster() *TestCluster + CloseShard(namespaceID string, workflowID string) + OverrideDynamicConfig(setting dynamicconfig.GenericSetting, value any) (cleanup func()) +} + +type testEnv struct { + *FunctionalTestBase + *require.Assertions + historyrequire.HistoryRequire + + Logger log.Logger + + cluster *TestCluster + nsName namespace.Name + taskPoller *taskpoller.TaskPoller + t *testing.T + tv *testvars.TestVars +} + +type TestOption func(*testOptions) + +type testOptions struct { + dedicatedCluster bool + dynamicConfigSettings []dynamicConfigOverride +} + +type dynamicConfigOverride struct { + setting dynamicconfig.GenericSetting + value any +} + +// WithDedicatedCluster requests a dedicated (non-shared) cluster for the test. +// Use this for tests that have cluster-global side effects. +func WithDedicatedCluster() TestOption { + return func(o *testOptions) { + o.dedicatedCluster = true + } +} + +// WithDynamicConfig overrides a dynamic config setting for the test. +// For settings that can be namespace-scoped, a namespace constraint is applied. +// For all others that require a dedicated cluster, this implies `WithDedicatedCluster`. +func WithDynamicConfig(setting dynamicconfig.GenericSetting, value any) TestOption { + return func(o *testOptions) { + if err := setting.Validate(value); err != nil { + panic(fmt.Sprintf("invalid value for setting %s: %v", setting.Key(), err)) + } + if !canBeNamespaceScoped(setting.Precedence()) { + o.dedicatedCluster = true + } + o.dynamicConfigSettings = append(o.dynamicConfigSettings, dynamicConfigOverride{setting: setting, value: value}) + } +} + +// NewEnv creates a new test environment with access to a Temporal cluster. +// The test is automatically marked as parallel. +func NewEnv(t *testing.T, opts ...TestOption) *testEnv { + t.Parallel() + + var options testOptions + for _, opt := range opts { + opt(&options) + } + + // For dedicated clusters, pass all dynamic config settings at cluster creation. + var startupConfig map[dynamicconfig.Key]any + if options.dedicatedCluster && len(options.dynamicConfigSettings) > 0 { + startupConfig = make(map[dynamicconfig.Key]any, len(options.dynamicConfigSettings)) + for _, override := range options.dynamicConfigSettings { + startupConfig[override.setting.Key()] = override.value + } + } + + base := testClusterPool.get(t, options.dedicatedCluster, startupConfig) + cluster := base.GetTestCluster() + + // Create a dedicated namespace for the test to help with test isolation. + ns := namespace.Name(RandomizeStr(t.Name())) + if _, err := base.RegisterNamespace( + ns, + 1, // 1 day retention + enumspb.ARCHIVAL_STATE_DISABLED, + "", + "", + ); err != nil { + t.Fatalf("Failed to register namespace: %v", err) + } + + env := &testEnv{ + FunctionalTestBase: base, + Assertions: require.New(t), + HistoryRequire: historyrequire.New(t), + cluster: cluster, + nsName: ns, + Logger: base.Logger, + taskPoller: taskpoller.New(t, cluster.FrontendClient(), ns.String()), + t: t, + tv: testvars.New(t), + } + + // For shared clusters, apply all dynamic config settings as overrides. + if !options.dedicatedCluster && len(options.dynamicConfigSettings) > 0 { + for _, override := range options.dynamicConfigSettings { + env.OverrideDynamicConfig(override.setting, override.value) + } + } + + return env +} + +// Use test env-specific namespace here for test isolation. +func (e *testEnv) Namespace() namespace.Name { + return e.nsName +} + +func (e *testEnv) TaskPoller() *taskpoller.TaskPoller { + return e.taskPoller +} + +func (e *testEnv) T() *testing.T { + return e.t +} + +func (e *testEnv) Tv() *testvars.TestVars { + return e.tv +} + +// OverrideDynamicConfig overrides a dynamic config setting for the duration of this test. +// For settings that can be namespace-scoped, a namespace constraint is applied. +// All others cannot be applied to a shared cluster and require `WithDedicatedCluster`. +func (e *testEnv) OverrideDynamicConfig(setting dynamicconfig.GenericSetting, value any) (cleanup func()) { + if e.isShared { + if !canBeNamespaceScoped(setting.Precedence()) { + e.t.Fatalf("OverrideDynamicConfig for setting %s (precedence %v) cannot be called on a shared cluster; use testcore.WithDedicatedCluster()", setting.Key(), setting.Precedence()) + } + + // Wrap value with namespace constraint for test isolation on shared clusters. + ns := e.nsName.String() + if cvs, ok := value.([]dynamicconfig.ConstrainedValue); ok { + result := make([]dynamicconfig.ConstrainedValue, len(cvs)) + for i, cv := range cvs { + cv.Constraints.Namespace = ns + result[i] = cv + } + value = result + } else { + value = []dynamicconfig.ConstrainedValue{{ + Constraints: dynamicconfig.Constraints{Namespace: ns}, + Value: value, + }} + } + } + return e.cluster.host.overrideDynamicConfig(e.t, setting.Key(), value) +} + +func canBeNamespaceScoped(p dynamicconfig.Precedence) bool { + switch p { + case dynamicconfig.PrecedenceNamespace, + dynamicconfig.PrecedenceTaskQueue, + dynamicconfig.PrecedenceDestination: + return true + default: + return false + } +} diff --git a/tests/update_workflow_test.go b/tests/update_workflow_test.go index 7ab820da11..70059cd164 100644 --- a/tests/update_workflow_test.go +++ b/tests/update_workflow_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/stretchr/testify/suite" + "github.com/stretchr/testify/assert" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -33,31 +33,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" ) -type UpdateWorkflowSuite struct { - testcore.FunctionalTestBase -} - -func TestUpdateWorkflowSuite(t *testing.T) { - t.Parallel() - s := new(UpdateWorkflowSuite) - suite.Run(t, s) -} - -// Simulating a graceful shard closure. The shard finalizer will clear the workflow context, -// any update requests are aborted and the frontend retries any in-flight update requests. -func (s *UpdateWorkflowSuite) clearUpdateRegistryAndAbortPendingUpdates(tv *testvars.TestVars) { - s.closeShard(tv.WorkflowID()) -} - -// Simulating an unexpected loss of the update registry due to a crash. The shard finalizer won't run, -// therefore the workflow context is NOT cleared, pending update requests are NOT aborted and will time out. -func (s *UpdateWorkflowSuite) loseUpdateRegistryAndAbandonPendingUpdates(tv *testvars.TestVars) { - cleanup := s.OverrideDynamicConfig(dynamicconfig.ShardFinalizerTimeout, 0) - defer cleanup() - s.closeShard(tv.WorkflowID()) -} - -func (s *UpdateWorkflowSuite) speculativeWorkflowTaskOutcomes( +func speculativeWorkflowTaskOutcomes( snap map[string][]*metricstest.CapturedRecording, ) (commits, rollbacks int) { for range snap[metrics.SpeculativeWorkflowTaskCommits.Name()] { @@ -69,39 +45,68 @@ func (s *UpdateWorkflowSuite) speculativeWorkflowTaskOutcomes( return } -func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_AcceptComplete() { - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, +func clearUpdateRegistryAndAbortPendingUpdates(s testcore.Env, tv *testvars.TestVars) { + closeShard(s, tv.WorkflowID()) +} + +func loseUpdateRegistryAndAbandonPendingUpdates(s testcore.Env, tv *testvars.TestVars) { + cleanup := s.OverrideDynamicConfig(dynamicconfig.ShardFinalizerTimeout, 0) + defer cleanup() + closeShard(s, tv.WorkflowID()) +} + +func closeShard(s testcore.Env, wid string) { + s.T().Helper() + + resp, err := s.FrontendClient().DescribeNamespace(testcore.NewContext(), &workflowservice.DescribeNamespaceRequest{ + Namespace: s.Namespace().String(), + }) + if err != nil { + s.T().Fatalf("Failed to describe namespace: %v", err) } - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) + s.CloseShard(resp.NamespaceInfo.Id, wid) +} - runID := mustStartWorkflow(s, tv) +func TestWorkflowUpdateSuite(t *testing.T) { + t.Run("EmptySpeculativeWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } - capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() - defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Uses CaptureMetricsHandler which requires a dedicated cluster to avoid metric interference. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + runID := mustStartWorkflow(s, s.Tv()) - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } + + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -109,130 +114,134 @@ func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_AcceptComplete() 5 WorkflowTaskScheduled // Speculative WT events are not written to the history yet. 6 WorkflowTaskStarted `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) - - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res.NewTask) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) - - // Test non-blocking poll - for _, waitPolicy := range []*updatepb.WaitPolicy{{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_UNSPECIFIED}, nil} { - pollUpdateResp, err := pollUpdate(s, tv, waitPolicy) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() s.NoError(err) - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, pollUpdateResp.Stage) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), pollUpdateResp.Outcome.GetSuccess())) - // Even if tv doesn't have RunID, it should be returned as part of UpdateRef. - s.Equal(runID, pollUpdateResp.UpdateRef.GetWorkflowExecution().RunId) - } - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + updateResultCh := sendUpdateNoError(s, tv) - commits, rollbacks := s.speculativeWorkflowTaskOutcomes(capture.Snapshot()) - s.Equal(1, commits) - s.Equal(0, rollbacks) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res.NewTask) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - events := s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution()) + // Test non-blocking poll + for _, waitPolicy := range []*updatepb.WaitPolicy{{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_UNSPECIFIED}, nil} { + pollUpdateResp, err := pollUpdate(s, s.Tv(), waitPolicy) + s.NoError(err) + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, pollUpdateResp.Stage) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), pollUpdateResp.Outcome.GetSuccess())) + // Even if tv doesn't have RunID, it should be returned as part of UpdateRef. + s.Equal(runID, pollUpdateResp.UpdateRef.GetWorkflowExecution().RunId) + } - s.EqualHistoryEvents(` + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + commits, rollbacks := speculativeWorkflowTaskOutcomes(capture.Snapshot()) + s.Equal(1, commits) + s.Equal(0, rollbacks) + + events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Was speculative WT... 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted // ...and events were written to the history when WT completes. + 7 WorkflowTaskCompleted // ...and events were written to the history when WT completes. 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} `, events) - }) - } -} + }) + } + }) -func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_AcceptComplete() { - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, - } + t.Run("NotEmptySpeculativeWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } - runID := mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with update unrelated command. - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv.ActivityID(), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: tv.Any().InfiniteTimeout(), - }}, - }}, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with update unrelated command. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: s.Tv().ActivityID(), + ActivityType: s.Tv().ActivityType(), + TaskQueue: s.Tv().TaskQueue(), + ScheduleToCloseTimeout: s.Tv().Any().InfiniteTimeout(), + }}, + }}, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -241,65 +250,66 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_AcceptComplete 6 WorkflowTaskScheduled // Speculative WFT with ActivityTaskScheduled(5) event after WorkflowTaskCompleted(4). 7 WorkflowTaskStarted `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(6, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(6, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) + updateResultCh := sendUpdateNoError(s, tv) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -307,100 +317,103 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_AcceptComplete 5 ActivityTaskScheduled 6 WorkflowTaskScheduled // Speculative WFT was persisted when completed (event 8) 7 WorkflowTaskStarted - 8 WorkflowTaskCompleted + 8 WorkflowTaskCompleted 9 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 6} // WTScheduled event which delivered update to the worker. 10 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 9} `, events) - }) - } -} + }) + } + }) -func (s *UpdateWorkflowSuite) TestFirstNormalScheduledWorkflowTask_AcceptComplete() { + t.Run("FirstNormalScheduledWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, - } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) - - runID := mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted // First normal WT. No speculative WT was created. `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(2, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(2, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - updateResultCh := sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) + updateResultCh := sendUpdateNoError(s, tv) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(1, wtHandlerCalls) - s.Equal(1, msgHandlerCalls) + s.Equal(1, wtHandlerCalls) + s.Equal(1, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -408,41 +421,43 @@ func (s *UpdateWorkflowSuite) TestFirstNormalScheduledWorkflowTask_AcceptComplet 5 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 2} // WTScheduled event which delivered update to the worker. 6 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 5} `, events) - }) - } -} + }) + } + }) -func (s *UpdateWorkflowSuite) TestNormalScheduledWorkflowTask_AcceptComplete() { + t.Run("NormalScheduledWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, - } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) - - runID := mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -450,71 +465,72 @@ func (s *UpdateWorkflowSuite) TestNormalScheduledWorkflowTask_AcceptComplete() { 5 WorkflowExecutionSignaled 6 WorkflowTaskScheduled // This WT was already created by signal and no speculative WT was created. 7 WorkflowTaskStarted`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Require().True(len(task.Messages) > 0, "Task has no messages", task) - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Require().NotEmpty(task.Messages, "Task has no messages", task) + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(6, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(6, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - // Send signal to schedule new WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) + // Send signal to schedule new WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - updateResultCh := sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) + updateResultCh := sendUpdateNoError(s, tv) - // Process update in workflow. It will be attached to existing WT. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) + // Process update in workflow. It will be attached to existing WT. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -526,38 +542,38 @@ func (s *UpdateWorkflowSuite) TestNormalScheduledWorkflowTask_AcceptComplete() { 9 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 6} // WTScheduled event which delivered update to the worker. 10 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 9} `, events) - }) - } -} + }) + } + }) -func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewEmptySpeculativeWorkflowTask_Rejected() { - tv := testvars.New(s.T()) + t.Run("RunningWorkflowTask_NewEmptySpeculativeWorkflowTask_Rejected", func(t *testing.T) { + // Uses CaptureMetricsHandler which requires a dedicated cluster to avoid metric interference. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() - defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) + var updateResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - var updateResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Send update after 1st WT has started. - updateResultCh = sendUpdateNoError(s, tv) - // Completes WT with empty command list to create next WFT w/o events. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Send update after 1st WT has started. + updateResultCh = sendUpdateNoError(s, s.Tv()) + // Completes WT with empty command list to create next WFT w/o events. + return nil, nil + case 2: + s.EqualHistory(` 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Speculative WT2 which was created while completing WT1. 6 WorkflowTaskStarted`, task.History) - // Message handler rejects update. - return nil, nil - case 3: - s.EqualHistory(` + // Message handler rejects update. + return nil, nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -565,79 +581,78 @@ func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewEmptySpeculativeWorkflo 5 WorkflowExecutionSignaled 6 WorkflowTaskScheduled 7 WorkflowTaskStarted`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1) - updRequestMsg := task.Messages[0] - s.EqualValues(5, updRequestMsg.GetEventId()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1, 3: + return nil, nil + case 2: + s.Len(task.Messages, 1) + updRequestMsg := task.Messages[0] + s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - case 3: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT which starts 1st update. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - wt1Resp := res.NewTask + // Drain first WT which starts 1st update. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + wt1Resp := res.NewTask - // Reject update in 2nd WT. - wt2Resp, err := poller.HandlePartialWorkflowTask(wt1Resp.GetWorkflowTask(), false) - s.NoError(err) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(3, wt2Resp.ResetHistoryEventId) + // Reject update in 2nd WT. + wt2Resp, err := poller.HandlePartialWorkflowTask(wt1Resp.GetWorkflowTask(), false) + s.NoError(err) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(3, wt2Resp.ResetHistoryEventId) - // Send signal to create WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) + // Send signal to create WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(completeWorkflowResp) + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(completeWorkflowResp) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - commits, rollbacks := s.speculativeWorkflowTaskOutcomes(capture.Snapshot()) - s.Equal(0, commits) - s.Equal(1, rollbacks) + commits, rollbacks := speculativeWorkflowTaskOutcomes(capture.Snapshot()) + s.Equal(0, commits) + s.Equal(1, rollbacks) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -647,43 +662,42 @@ func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewEmptySpeculativeWorkflo 7 WorkflowTaskStarted 8 WorkflowTaskCompleted 9 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewNotEmptySpeculativeWorkflowTask_Rejected() { + t.Run("RunningWorkflowTask_NewNotEmptySpeculativeWorkflowTask_Rejected", func(t *testing.T) { + s := testcore.NewEnv(t) - tv := testvars.New(s.T()) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + var updateResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - var updateResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Send update after 1st WT has started. - updateResultCh = sendUpdateNoError(s, tv) - // Completes WT with update unrelated commands to create events that will be in the next speculative WFT. - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv.ActivityID(), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: tv.Any().InfiniteTimeout(), - }}, - }}, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Send update after 1st WT has started. + updateResultCh = sendUpdateNoError(s, s.Tv()) + // Completes WT with update unrelated commands to create events that will be in the next speculative WFT. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: s.Tv().ActivityID(), + ActivityType: s.Tv().ActivityType(), + TaskQueue: s.Tv().TaskQueue(), + ScheduleToCloseTimeout: s.Tv().Any().InfiniteTimeout(), + }}, + }}, nil + case 2: + s.EqualHistory(` 4 WorkflowTaskCompleted 5 ActivityTaskScheduled 6 WorkflowTaskScheduled // Speculative WFT2 with event (5) which was created while completing WFT1. 7 WorkflowTaskStarted`, task.History) - // Message handler rejects update. - return nil, nil - case 3: - s.EqualHistory(` + // Message handler rejects update. + return nil, nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -698,81 +712,80 @@ func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewNotEmptySpeculativeWork 12 WorkflowTaskStarted `, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1) - updRequestMsg := task.Messages[0] - s.EqualValues(6, updRequestMsg.GetEventId()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1, 3: + return nil, nil + case 2: + s.Len(task.Messages, 1) + updRequestMsg := task.Messages[0] + s.EqualValues(6, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - case 3: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT which starts 1st update. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - wt1Resp := res.NewTask + // Drain first WT which starts 1st update. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + wt1Resp := res.NewTask - // Reject update in 2nd WT. - wt2Resp, err := poller.HandlePartialWorkflowTask(wt1Resp.GetWorkflowTask(), false) - s.NoError(err) - s.NotNil(wt2Resp) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, wt2Resp.ResetHistoryEventId) + // Reject update in 2nd WT. + wt2Resp, err := poller.HandlePartialWorkflowTask(wt1Resp.GetWorkflowTask(), false) + s.NoError(err) + s.NotNil(wt2Resp) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, wt2Resp.ResetHistoryEventId) - // Schedule new WFT. - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + // Schedule new WFT. + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(completeWorkflowResp) - s.EqualValues(0, completeWorkflowResp.NewTask.ResetHistoryEventId) + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(completeWorkflowResp) + s.EqualValues(0, completeWorkflowResp.NewTask.ResetHistoryEventId) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -787,574 +800,582 @@ func (s *UpdateWorkflowSuite) TestRunningWorkflowTask_NewNotEmptySpeculativeWork 12 WorkflowTaskStarted 13 WorkflowTaskCompleted 14 WorkflowExecutionCompleted`, events) -} - -func (s *UpdateWorkflowSuite) TestCompletedWorkflow() { - s.Run("receive outcome from completed Update", func() { - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - res := s.UpdateAcceptCompleteCommands(tv) - res = append(res, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }) - return res, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - return s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - - // Send Update request. - updateResultCh := sendUpdateNoError(s, tv) - - // Complete Update and Workflow. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - - // Receive Update result. - updateResult1 := <-updateResultCh - s.NotNil(updateResult1.GetOutcome().GetSuccess()) - - // Send same Update request again, receiving the same Update result. - updateResultCh = sendUpdateNoError(s, tv) - updateResult2 := <-updateResultCh - s.EqualValues(updateResult1, updateResult2) }) - s.Run("receive update failure from accepted Update", func() { - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - res := s.UpdateAcceptCommands(tv) - res = append(res, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }) - return res, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - return s.UpdateAcceptMessages(tv, task.Messages[0]), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - - // Send Update request. - updateResultCh := sendUpdate(testcore.NewContext(), s, tv) - - // Accept Update and complete Workflow. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - - // Receive Update result. - updateResult1 := <-updateResultCh - s.NoError(updateResult1.err) - s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult1.response.GetOutcome().GetFailure().GetMessage()) - - // Send same Update request again, receiving the same failure. - updateResultCh = sendUpdate(testcore.NewContext(), s, tv) - updateResult2 := <-updateResultCh - s.NoError(updateResult2.err) - s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult2.response.GetOutcome().GetFailure().GetMessage()) - }) -} - -func (s *UpdateWorkflowSuite) TestValidateWorkerMessages() { - testCases := []struct { - Name string - RespondWorkflowTaskError string - MessageFn func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message - CommandFn func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command - }{ - { - Name: "message-update-id-not-found-and-accepted-request-not-set", - RespondWorkflowTaskError: "wasn't found", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: tv.UpdateID() + tv.Any().String(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: nil, // Important not to pass original request back. - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - } - }, - }, - { - Name: "message-update-id-not-found-and-accepted-request-is-set", - RespondWorkflowTaskError: "", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: tv.UpdateID() + tv.Any().String(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, // Update will be resurrected from original request. - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - } - }, - }, - { - Name: "command-reference-missed-message", - RespondWorkflowTaskError: "referenced absent message ID", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.Any().String(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - } - }, - }, - { - Name: "complete-without-accept", - RespondWorkflowTaskError: "invalid state transition attempted", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-completed", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Response{ - Meta: updRequest.GetMeta(), - Outcome: &updatepb.Outcome{ - Value: &updatepb.Outcome_Success{ - Success: tv.Any().Payloads(), - }, - }, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-completed", - }}, - }, - } - }, - }, - { - Name: "accept-twice", - RespondWorkflowTaskError: "invalid state transition attempted", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.WithMessageIDNumber(1).MessageID(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - { - Id: tv.WithMessageIDNumber(2).MessageID(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.WithMessageIDNumber(1).MessageID(), - }}, - }, - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.WithMessageIDNumber(2).MessageID(), - }}, - }, - } - }, - }, - { - Name: "success-case", - RespondWorkflowTaskError: "", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - { - Id: tv.MessageID() + "_update-completed", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Response{ - Meta: updRequest.GetMeta(), - Outcome: &updatepb.Outcome{ - Value: &updatepb.Outcome_Success{ - Success: tv.Any().Payloads(), - }, - }, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-completed", - }}, - }, - } - }, - }, - { - Name: "success-case-no-commands", // PROTOCOL_MESSAGE commands are optional. - RespondWorkflowTaskError: "", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.Any().String(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - { - Id: tv.Any().String(), - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Response{ - Meta: updRequest.GetMeta(), - Outcome: &updatepb.Outcome{ - Value: &updatepb.Outcome_Success{ - Success: tv.Any().Payloads(), - }, - }, - }), - }, - } - }, - }, - { - Name: "invalid-command-order", - RespondWorkflowTaskError: "invalid state transition attempted", - MessageFn: func(tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), reqMsg.GetBody()) - return []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: reqMsg.GetId(), - AcceptedRequestSequencingEventId: reqMsg.GetEventId(), - AcceptedRequest: updRequest, - }), - }, - { - Id: tv.MessageID() + "_update-completed", - ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Response{ - Meta: updRequest.GetMeta(), - Outcome: &updatepb.Outcome{ - Value: &updatepb.Outcome_Success{ - Success: tv.Any().Payloads(), - }, - }, - }), - }, - } - }, - CommandFn: func(tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { - return []*commandpb.Command{ - // Complete command goes before Accept command. - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-completed", - }}, - }, - { - CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, - Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ - MessageId: tv.MessageID() + "_update-accepted", - }}, - }, - } - }, - }, - } - - for _, tc := range testCases { - s.Run(tc.Name, func() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) + t.Run("CompletedWorkflow", func(t *testing.T) { + t.Run("receive outcome from completed Update", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + wtHandlerCalls := 0 wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - if tc.CommandFn == nil { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + res := s.UpdateAcceptCompleteCommands(s.Tv()) + res = append(res, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }) + return res, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) return nil, nil } - return tc.CommandFn(tv, task.History), nil } + msgHandlerCalls := 0 msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - if tc.MessageFn == nil { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + return s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) return nil, nil } - updRequestMsg := task.Messages[0] - return tc.MessageFn(tv, updRequestMsg), nil } + //nolint:staticcheck // SA1019 TaskPoller replacement needed poller := &testcore.TaskPoller{ Client: s.FrontendClient(), Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), WorkflowTaskHandler: wtHandler, MessageHandler: msgHandler, Logger: s.Logger, T: s.T(), } - halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) - defer cancel() - updateResultCh := sendUpdate(halfSecondTimeoutCtx, s, tv) - - // Process update in workflow. - _, err := poller.PollAndProcessWorkflowTask() - updateResult := <-updateResultCh - if tc.RespondWorkflowTaskError != "" { - s.Error(err, "RespondWorkflowTaskCompleted should return an error contains `%v`", tc.RespondWorkflowTaskError) - s.Contains(err.Error(), tc.RespondWorkflowTaskError) - - var wfNotReady *serviceerror.WorkflowNotReady - s.ErrorAs(updateResult.err, &wfNotReady, "API caller should get serviceerror.WorkflowNotReady, if server got a validation error while processing worker response.") - s.Contains(updateResult.err.Error(), "Unable to perform workflow execution update due to unexpected workflow task failure.") - s.Nil(updateResult.response) - } else { - s.NoError(err) - s.NoError(updateResult.err) - } - }) - } -} - -func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete() { - testCases := []struct { - name string - useRunID bool - }{ - { - name: "with RunID", - useRunID: true, - }, - { - name: "without RunID", - useRunID: false, - }, - } - - for _, tc := range testCases { - s.Run(tc.name, func() { - tv := testvars.New(s.T()) - - runID := mustStartWorkflow(s, tv) - - // Drain existing first WT from regular task queue, but respond with sticky queue enabled response, next WT will go to sticky queue. - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - StickyAttributes: tv.StickyExecutionAttributes(3 * time.Second), - }, nil - }) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) s.NoError(err) - go func() { - // Process update in workflow task (it is sticky). - res, err := s.TaskPoller(). - PollWorkflowTask(&workflowservice.PollWorkflowTaskQueueRequest{TaskQueue: tv.StickyTaskQueue()}). - HandleTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - // This WT contains partial history because sticky was enabled. - s.EqualHistory(` + // Send Update request. + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Complete Update and Workflow. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + + // Receive Update result. + updateResult1 := <-updateResultCh + s.NotNil(updateResult1.GetOutcome().GetSuccess()) + + // Send same Update request again, receiving the same Update result. + updateResultCh = sendUpdateNoError(s, s.Tv()) + updateResult2 := <-updateResultCh + s.Equal(updateResult1, updateResult2) + }) + + t.Run("receive update failure from accepted Update", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + res := s.UpdateAcceptCommands(s.Tv()) + res = append(res, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }) + return res, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + return s.UpdateAcceptMessages(s.Tv(), task.Messages[0]), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + + // Send Update request. + updateResultCh := sendUpdate(testcore.NewContext(), s, s.Tv()) + + // Accept Update and complete Workflow. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + + // Receive Update result. + updateResult1 := <-updateResultCh + s.NoError(updateResult1.err) + s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult1.response.GetOutcome().GetFailure().GetMessage()) + + // Send same Update request again, receiving the same failure. + updateResultCh = sendUpdate(testcore.NewContext(), s, s.Tv()) + updateResult2 := <-updateResultCh + s.NoError(updateResult2.err) + s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult2.response.GetOutcome().GetFailure().GetMessage()) + }) + }) + + t.Run("ValidateWorkerMessages", func(t *testing.T) { + testCases := []struct { + Name string + RespondWorkflowTaskError string + MessageFn func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message + CommandFn func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command + }{ + { + Name: "message-update-id-not-found-and-accepted-request-not-set", + RespondWorkflowTaskError: "wasn't found", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-accepted", + ProtocolInstanceId: tv.UpdateID() + tv.Any().String(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: nil, // Important not to pass original request back. + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + } + }, + }, + { + Name: "message-update-id-not-found-and-accepted-request-is-set", + RespondWorkflowTaskError: "", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-accepted", + ProtocolInstanceId: tv.UpdateID() + tv.Any().String(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, // Update will be resurrected from original request. + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + } + }, + }, + { + Name: "command-reference-missed-message", + RespondWorkflowTaskError: "referenced absent message ID", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.Any().String(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + } + }, + }, + { + Name: "complete-without-accept", + RespondWorkflowTaskError: "invalid state transition attempted", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-completed", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Response{ + Meta: updRequest.GetMeta(), + Outcome: &updatepb.Outcome{ + Value: &updatepb.Outcome_Success{ + Success: tv.Any().Payloads(), + }, + }, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-completed", + }}, + }, + } + }, + }, + { + Name: "accept-twice", + RespondWorkflowTaskError: "invalid state transition attempted", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.WithMessageIDNumber(1).MessageID(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + { + Id: tv.WithMessageIDNumber(2).MessageID(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.WithMessageIDNumber(1).MessageID(), + }}, + }, + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.WithMessageIDNumber(2).MessageID(), + }}, + }, + } + }, + }, + { + Name: "success-case", + RespondWorkflowTaskError: "", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-accepted", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + { + Id: tv.MessageID() + "_update-completed", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Response{ + Meta: updRequest.GetMeta(), + Outcome: &updatepb.Outcome{ + Value: &updatepb.Outcome_Success{ + Success: tv.Any().Payloads(), + }, + }, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-completed", + }}, + }, + } + }, + }, + { + Name: "success-case-no-commands", // PROTOCOL_MESSAGE commands are optional. + RespondWorkflowTaskError: "", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.Any().String(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + { + Id: tv.Any().String(), + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Response{ + Meta: updRequest.GetMeta(), + Outcome: &updatepb.Outcome{ + Value: &updatepb.Outcome_Success{ + Success: tv.Any().Payloads(), + }, + }, + }), + }, + } + }, + }, + { + Name: "invalid-command-order", + RespondWorkflowTaskError: "invalid state transition attempted", + MessageFn: func(t *testing.T, tv *testvars.TestVars, reqMsg *protocolpb.Message) []*protocolpb.Message { + updRequest := protoutils.UnmarshalAny[*updatepb.Request](t, reqMsg.GetBody()) + return []*protocolpb.Message{ + { + Id: tv.MessageID() + "_update-accepted", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Acceptance{ + AcceptedRequestMessageId: reqMsg.GetId(), + AcceptedRequestSequencingEventId: reqMsg.GetEventId(), + AcceptedRequest: updRequest, + }), + }, + { + Id: tv.MessageID() + "_update-completed", + ProtocolInstanceId: updRequest.GetMeta().GetUpdateId(), + SequencingId: nil, + Body: protoutils.MarshalAny(t, &updatepb.Response{ + Meta: updRequest.GetMeta(), + Outcome: &updatepb.Outcome{ + Value: &updatepb.Outcome_Success{ + Success: tv.Any().Payloads(), + }, + }, + }), + }, + } + }, + CommandFn: func(t *testing.T, tv *testvars.TestVars, history *historypb.History) []*commandpb.Command { + return []*commandpb.Command{ + // Complete command goes before Accept command. + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-completed", + }}, + }, + { + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{ + MessageId: tv.MessageID() + "_update-accepted", + }}, + }, + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + if tc.CommandFn == nil { + return nil, nil + } + return tc.CommandFn(t, s.Tv(), task.History), nil + } + + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + if tc.MessageFn == nil { + return nil, nil + } + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] + return tc.MessageFn(t, s.Tv(), updRequestMsg), nil + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: t, + } + + halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) + defer cancel() + updateResultCh := sendUpdate(halfSecondTimeoutCtx, s, s.Tv()) + + // Process update in workflow. + _, err := poller.PollAndProcessWorkflowTask() + updateResult := <-updateResultCh + if tc.RespondWorkflowTaskError != "" { + s.Error(err, "RespondWorkflowTaskCompleted should return an error contains `%v`", tc.RespondWorkflowTaskError) + s.Contains(err.Error(), tc.RespondWorkflowTaskError) + + var wfNotReady *serviceerror.WorkflowNotReady + s.ErrorAs(updateResult.err, &wfNotReady, "API caller should get serviceerror.WorkflowNotReady, if server got a validation error while processing worker response.") + s.Contains(updateResult.err.Error(), "Unable to perform workflow execution update due to unexpected workflow task failure.") + s.Nil(updateResult.response) + } else { + s.NoError(err) + s.NoError(updateResult.err) + } + }) + } + }) + + t.Run("StickySpeculativeWorkflowTask_AcceptComplete", func(t *testing.T) { + testCases := []struct { + name string + useRunID bool + }{ + { + name: "with RunID", + useRunID: true, + }, + { + name: "without RunID", + useRunID: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if tc.useRunID { + tv = tv.WithRunID(runID) + } + + // Drain existing first WT from regular task queue, but respond with sticky queue enabled response, next WT will go to sticky queue. + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + StickyAttributes: s.Tv().StickyExecutionAttributes(3 * time.Second), + }, nil + }) + s.NoError(err) + + go func() { + // Process update in workflow task (it is sticky). + res, err := s.TaskPoller(). + PollWorkflowTask(&workflowservice.PollWorkflowTaskQueueRequest{TaskQueue: s.Tv().StickyTaskQueue()}). + HandleTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + // This WT contains partial history because sticky was enabled. + s.EqualHistory(` 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted`, task.History) - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + //nolint:testifylint // callback runs synchronously within HandleTask + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) //nolint:testifylint // callback runs synchronously within HandleTask + s.EqualValues(5, updRequestMsg.GetEventId()) //nolint:testifylint // callback runs synchronously within HandleTask - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, updRequestMsg), - }, nil - }) - s.NoError(err) - s.NotNil(res) - s.EqualValues(0, res.ResetHistoryEventId) - }() + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), + }, nil + }) + //nolint:testifylint // intentional async polling pattern + s.NoError(err) + s.NotNil(res) //nolint:testifylint // intentional async polling pattern + s.EqualValues(0, res.ResetHistoryEventId) //nolint:testifylint // intentional async polling pattern + }() - // This is to make sure that sticky poller above reached server first. - // And when update comes, stick poller is already available. - time.Sleep(500 * time.Millisecond) //nolint:forbidigo - updateResult := <-sendUpdateNoError(s, s.useRunID(tv, tc.useRunID, runID)) + // This is to make sure that sticky poller above reached server first. + // And when update comes, stick poller is already available. + time.Sleep(500 * time.Millisecond) //nolint:forbidigo + updateResult := <-sendUpdateNoError(s, tv) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1364,26 +1385,25 @@ func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete() 7 WorkflowTaskCompleted 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} - `, s.GetHistory(s.Namespace().String(), s.useRunID(tv, tc.useRunID, runID).WorkflowExecution())) - }) - } -} + `, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + }) + } + }) -func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_StickyWorkerUnavailable() { - tv := testvars.New(s.T()) + t.Run("StickySpeculativeWorkflowTask_AcceptComplete_StickyWorkerUnavailable", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - // Worker gets full history because update was issued after sticky worker is gone. - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + // Worker gets full history because update was issued after sticky worker is gone. + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1391,74 +1411,75 @@ func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_S 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - StickyTaskQueue: tv.StickyTaskQueue(), - StickyScheduleToStartTimeout: 3 * time.Second, - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + StickyTaskQueue: s.Tv().StickyTaskQueue(), + StickyScheduleToStartTimeout: 3 * time.Second, + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain existing WT from regular task queue, but respond with sticky enabled response to enable stick task queue. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithRespondSticky, testcore.WithoutRetries) - s.NoError(err) + // Drain existing WT from regular task queue, but respond with sticky enabled response to enable stick task queue. + _, err := poller.PollAndProcessWorkflowTask(testcore.WithRespondSticky, testcore.WithoutRetries) + s.NoError(err) - s.Logger.Info("Sleep 10+ seconds to make sure stickyPollerUnavailableWindow time has passed.") - time.Sleep(10*time.Second + 100*time.Millisecond) //nolint:forbidigo - s.Logger.Info("Sleep 10+ seconds is done.") + s.Logger.Info("Sleep 10+ seconds to make sure stickyPollerUnavailableWindow time has passed.") + time.Sleep(10*time.Second + 100*time.Millisecond) //nolint:forbidigo + s.Logger.Info("Sleep 10+ seconds is done.") - // Now send an update. It should try sticky task queue first, but got "StickyWorkerUnavailable" error - // and resend it to normal. - // This can be observed in wtHandler: if history is partial => sticky task queue is used. - updateResultCh := sendUpdateNoError(s, tv) + // Now send an update. It should try sticky task queue first, but got "StickyWorkerUnavailable" error + // and resend it to normal. + // This can be observed in wtHandler: if history is partial => sticky task queue is used. + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Process update in workflow task from non-sticky task queue. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Process update in workflow task from non-sticky task queue. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1469,96 +1490,95 @@ func (s *UpdateWorkflowSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_S 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} `, events) -} + }) -func (s *UpdateWorkflowSuite) TestFirstNormalScheduledWorkflowTask_Reject() { - tv := testvars.New(s.T()) + t.Run("FirstNormalScheduledWorkflowTask_Reject", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted`, task.History) - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(2, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(2, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, updateResp.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, updateResp.ResetHistoryEventId) - s.Equal(1, wtHandlerCalls) - s.Equal(1, msgHandlerCalls) + s.Equal(1, wtHandlerCalls) + s.Equal(1, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled // First normal WT was scheduled before update and therefore all 3 events have to be written even if update was rejected. 3 WorkflowTaskStarted 4 WorkflowTaskCompleted // Empty completed WT. No new events were created after it. `, events) -} + }) -func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_Reject() { - tv := testvars.New(s.T()) + t.Run("EmptySpeculativeWorkflowTask_Reject", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + updateResultCh := sendUpdateNoError(s, s.Tv()) - updateResultCh := sendUpdateNoError(s, tv) - - // Process update in workflow. - res, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Process update in workflow. + res, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1567,30 +1587,30 @@ func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_Reject() { 6 WorkflowTaskStarted `, task.History) - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(5, updRequestMsg.GetEventId()) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, updRequestMsg), - }, nil - }) - s.NoError(err) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(3, res.ResetHistoryEventId) + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), updRequestMsg), + }, nil + }) + s.NoError(err) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(3, res.ResetHistoryEventId) - // Send signal to create WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) + // Send signal to create WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - // Process signal and complete workflow. - res, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Process signal and complete workflow. + res, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1599,22 +1619,22 @@ func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_Reject() { 6 WorkflowTaskScheduled 7 WorkflowTaskStarted`, task.History) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, }, }, - }, - }, nil - }) - s.NoError(err) - s.NotNil(res) + }, nil + }) + s.NoError(err) + s.NotNil(res) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1625,29 +1645,28 @@ func (s *UpdateWorkflowSuite) TestEmptySpeculativeWorkflowTask_Reject() { 8 WorkflowTaskCompleted 9 WorkflowExecutionCompleted `, events) -} + }) -func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_Reject() { - tv := testvars.New(s.T()) + t.Run("NotEmptySpeculativeWorkflowTask_Reject", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv.ActivityID(), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: tv.Any().InfiniteTimeout(), - }}, - }}, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: s.Tv().ActivityID(), + ActivityType: s.Tv().ActivityType(), + TaskQueue: s.Tv().TaskQueue(), + ScheduleToCloseTimeout: s.Tv().Any().InfiniteTimeout(), + }}, + }}, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1656,9 +1675,9 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_Reject() { 6 WorkflowTaskScheduled // Speculative WFT will be written to the history because there is ActivityTaskScheduled(5) event. 7 WorkflowTaskStarted `, task.History) - return nil, nil - case 3: - s.EqualHistory(` + return nil, nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1672,82 +1691,81 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_Reject() { 11 WorkflowTaskScheduled 12 WorkflowTaskStarted `, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1, 3: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(6, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(6, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - case 3: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with normal workflow task") + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with normal workflow task") - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete workflow. - res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) + // Complete workflow. + res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1762,35 +1780,34 @@ func (s *UpdateWorkflowSuite) TestNotEmptySpeculativeWorkflowTask_Reject() { 12 WorkflowTaskStarted 13 WorkflowTaskCompleted 14 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() { - tv := testvars.New(s.T()) + t.Run("1stAccept_2ndAccept_2ndComplete_1stComplete", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1).WithActivityIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2).WithActivityIDNumber(2) - mustStartWorkflow(s, tv) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1).WithActivityIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2).WithActivityIDNumber(2) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted`, task.History) - return append(s.UpdateAcceptCommands(tv1), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv1.ActivityID(), - ActivityType: tv1.ActivityType(), - TaskQueue: tv1.TaskQueue(), - ScheduleToCloseTimeout: tv1.Any().InfiniteTimeout(), - }}, - }), nil - case 2: - s.EqualHistory(` + return append(s.UpdateAcceptCommands(tv1), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv1.ActivityID(), + ActivityType: tv1.ActivityType(), + TaskQueue: tv1.TaskQueue(), + ScheduleToCloseTimeout: tv1.Any().InfiniteTimeout(), + }}, + }), nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1799,17 +1816,17 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() 6 ActivityTaskScheduled 7 WorkflowTaskScheduled // New normal WT is created because of the 2nd update. 8 WorkflowTaskStarted`, task.History) - return append(s.UpdateAcceptCommands(tv2), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv2.ActivityID(), - ActivityType: tv2.ActivityType(), - TaskQueue: tv2.TaskQueue(), - ScheduleToCloseTimeout: tv2.Any().InfiniteTimeout(), - }}, - }), nil - case 3: - s.EqualHistory(` + return append(s.UpdateAcceptCommands(tv2), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv2.ActivityID(), + ActivityType: tv2.ActivityType(), + TaskQueue: tv2.TaskQueue(), + ScheduleToCloseTimeout: tv2.Any().InfiniteTimeout(), + }}, + }), nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1826,9 +1843,9 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() 14 WorkflowTaskScheduled 15 WorkflowTaskStarted `, task.History) - return s.UpdateCompleteCommands(tv2), nil - case 4: - s.EqualHistory(` + return s.UpdateCompleteCommands(tv2), nil + case 4: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1851,103 +1868,104 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() 20 WorkflowTaskScheduled 21 WorkflowTaskStarted `, task.History) - return s.UpdateCompleteCommands(tv1), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateCompleteCommands(tv1), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - var upd1RequestMsg, upd2RequestMsg *protocolpb.Message + var upd1RequestMsg, upd2RequestMsg *protocolpb.Message - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - upd1RequestMsg = task.Messages[0] - upd1Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd1RequestMsg.GetBody()) - s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), upd1Request.GetInput().GetArgs())) - s.EqualValues(2, upd1RequestMsg.GetEventId()) - return s.UpdateAcceptMessages(tv1, upd1RequestMsg), nil - case 2: - upd2RequestMsg = task.Messages[0] - upd2Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd2RequestMsg.GetBody()) - s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), upd2Request.GetInput().GetArgs())) - s.EqualValues(7, upd2RequestMsg.GetEventId()) - return s.UpdateAcceptMessages(tv2, upd2RequestMsg), nil - case 3: - s.NotNil(upd2RequestMsg) - return s.UpdateCompleteMessages(tv2, upd2RequestMsg), nil - case 4: - s.NotNil(upd1RequestMsg) - return s.UpdateCompleteMessages(tv1, upd1RequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + upd1RequestMsg = task.Messages[0] + upd1Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd1RequestMsg.GetBody()) + s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), upd1Request.GetInput().GetArgs())) + s.EqualValues(2, upd1RequestMsg.GetEventId()) + return s.UpdateAcceptMessages(tv1, upd1RequestMsg), nil + case 2: + upd2RequestMsg = task.Messages[0] + upd2Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd2RequestMsg.GetBody()) + s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), upd2Request.GetInput().GetArgs())) + s.EqualValues(7, upd2RequestMsg.GetEventId()) + return s.UpdateAcceptMessages(tv2, upd2RequestMsg), nil + case 3: + s.NotNil(upd2RequestMsg) + return s.UpdateCompleteMessages(tv2, upd2RequestMsg), nil + case 4: + s.NotNil(upd1RequestMsg) + return s.UpdateCompleteMessages(tv1, upd1RequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } - updateResultCh1 := sendUpdateNoError(s, tv1) + updateResultCh1 := sendUpdateNoError(s, tv1) - // Accept update1 in normal WT1. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Accept update1 in normal WT1. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - // Send 2nd update and create speculative WT2. - updateResultCh2 := sendUpdateNoError(s, tv2) + // Send 2nd update and create speculative WT2. + updateResultCh2 := sendUpdateNoError(s, tv2) - // Poll for WT2 which 2nd update. Accept update2. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Poll for WT2 which 2nd update. Accept update2. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete update2 in WT3. - res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult2 := <-updateResultCh2 - s.EqualValues("success-result-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Complete update2 in WT3. + res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult2 := <-updateResultCh2 + s.Equal("success-result-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete update1 in WT4. - res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult1 := <-updateResultCh1 - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult1.Stage) - s.EqualValues("success-result-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updateResult1.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Complete update1 in WT4. + res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult1 := <-updateResultCh1 + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult1.Stage) + s.Equal("success-result-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updateResult1.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(4, wtHandlerCalls) - s.Equal(4, msgHandlerCalls) + s.Equal(4, wtHandlerCalls) + s.Equal(4, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -1972,36 +1990,35 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndAccept_2ndComplete_1stComplete() 22 WorkflowTaskCompleted 23 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 5} // 1st update is completed. `, events) -} + }) -func (s *UpdateWorkflowSuite) Test1stAccept_2ndReject_1stComplete() { - tv := testvars.New(s.T()) + t.Run("1stAccept_2ndReject_1stComplete", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1).WithActivityIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2).WithActivityIDNumber(2) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1).WithActivityIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2).WithActivityIDNumber(2) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted`, task.History) - return append(s.UpdateAcceptCommands(tv1), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv1.ActivityID(), - ActivityType: tv1.ActivityType(), - TaskQueue: tv1.TaskQueue(), - ScheduleToCloseTimeout: tv1.Any().InfiniteTimeout(), - }}, - }), nil - case 2: - s.EqualHistory(` + return append(s.UpdateAcceptCommands(tv1), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv1.ActivityID(), + ActivityType: tv1.ActivityType(), + TaskQueue: tv1.TaskQueue(), + ScheduleToCloseTimeout: tv1.Any().InfiniteTimeout(), + }}, + }), nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2011,10 +2028,10 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndReject_1stComplete() { 7 WorkflowTaskScheduled // Speculative WFT with WorkflowExecutionUpdateAccepted(5) event. 8 WorkflowTaskStarted `, task.History) - // Message handler rejects 2nd update. - return nil, nil - case 3: - s.EqualHistory(` + // Message handler rejects 2nd update. + return nil, nil + case 3: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2029,90 +2046,91 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndReject_1stComplete() { 12 WorkflowTaskScheduled 13 WorkflowTaskStarted `, task.History) - return s.UpdateCompleteCommands(tv1), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateCompleteCommands(tv1), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - var upd1RequestMsg *protocolpb.Message - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - upd1RequestMsg = task.Messages[0] - upd1Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd1RequestMsg.GetBody()) - s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), upd1Request.GetInput().GetArgs())) - s.EqualValues(2, upd1RequestMsg.GetEventId()) - return s.UpdateAcceptMessages(tv1, upd1RequestMsg), nil - case 2: - upd2RequestMsg := task.Messages[0] - upd2Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd2RequestMsg.GetBody()) - s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), upd2Request.GetInput().GetArgs())) - s.EqualValues(7, upd2RequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv2, upd2RequestMsg), nil - case 3: - s.NotNil(upd1RequestMsg) - return s.UpdateCompleteMessages(tv1, upd1RequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + var upd1RequestMsg *protocolpb.Message + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + upd1RequestMsg = task.Messages[0] + upd1Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd1RequestMsg.GetBody()) + s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), upd1Request.GetInput().GetArgs())) + s.EqualValues(2, upd1RequestMsg.GetEventId()) + return s.UpdateAcceptMessages(tv1, upd1RequestMsg), nil + case 2: + upd2RequestMsg := task.Messages[0] + upd2Request := protoutils.UnmarshalAny[*updatepb.Request](s.T(), upd2RequestMsg.GetBody()) + s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), upd2Request.GetInput().GetArgs())) + s.EqualValues(7, upd2RequestMsg.GetEventId()) + return s.UpdateRejectMessages(tv2, upd2RequestMsg), nil + case 3: + s.NotNil(upd1RequestMsg) + return s.UpdateCompleteMessages(tv1, upd1RequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } - updateResultCh1 := sendUpdateNoError(s, tv1) + updateResultCh1 := sendUpdateNoError(s, tv1) - // Accept update1 in WT1. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Accept update1 in WT1. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - // Send 2nd update and create speculative WT2. - updateResultCh2 := sendUpdateNoError(s, tv2) + // Send 2nd update and create speculative WT2. + updateResultCh2 := sendUpdateNoError(s, tv2) - // Poll for WT2 which 2nd update. Reject update2. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with workflow task which had events") + // Poll for WT2 which 2nd update. Reject update2. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with workflow task which had events") - updateResult2 := <-updateResultCh2 - s.Equal("rejection-of-"+tv2.UpdateID(), updateResult2.GetOutcome().GetFailure().GetMessage()) + updateResult2 := <-updateResultCh2 + s.Equal("rejection-of-"+tv2.UpdateID(), updateResult2.GetOutcome().GetFailure().GetMessage()) - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Complete update1 in WT3. - res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult1 := <-updateResultCh1 - s.EqualValues("success-result-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updateResult1.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Complete update1 in WT3. + res, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult1 := <-updateResultCh1 + s.Equal("success-result-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updateResult1.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2129,24 +2147,23 @@ func (s *UpdateWorkflowSuite) Test1stAccept_2ndReject_1stComplete() { 14 WorkflowTaskCompleted 15 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 5} `, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_Fail", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + timeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 2*time.Second) + defer cancel() + updateResultCh := sendUpdate(timeoutCtx, s, s.Tv()) - timeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 2*time.Second) - defer cancel() - updateResultCh := sendUpdate(timeoutCtx, s, tv) - - // Try to accept update in workflow: get malformed response. - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Try to accept update in workflow: get malformed response. + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2155,36 +2172,37 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 6 WorkflowTaskStarted `, task.History) - updRequestMsg := task.Messages[0] - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: s.UpdateAcceptCommands(tv), - // Emulate bug in worker/SDK update handler code. Return malformed acceptance response. - Messages: []*protocolpb.Message{ - { - Id: tv.MessageID() + "_update-accepted", - ProtocolInstanceId: tv.Any().String(), - SequencingId: nil, - Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ - AcceptedRequestMessageId: updRequestMsg.GetId(), - AcceptedRequestSequencingEventId: updRequestMsg.GetEventId(), - AcceptedRequest: nil, // must not be nil! - }), + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: s.UpdateAcceptCommands(s.Tv()), + // Emulate bug in worker/SDK update handler code. Return malformed acceptance response. + Messages: []*protocolpb.Message{ + { + Id: s.Tv().MessageID() + "_update-accepted", + ProtocolInstanceId: s.Tv().Any().String(), + SequencingId: nil, + Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ + AcceptedRequestMessageId: updRequestMsg.GetId(), + AcceptedRequestSequencingEventId: updRequestMsg.GetEventId(), + AcceptedRequest: nil, // must not be nil! + }), + }, }, - }, - }, nil - }) - s.Error(err) - s.Contains(err.Error(), "wasn't found") + }, nil + }) + s.Error(err) + s.Contains(err.Error(), "wasn't found") - // Update is aborted, speculative WFT failure is recorded into the history. - updateResult := <-updateResultCh - var wfNotReady *serviceerror.WorkflowNotReady - s.ErrorAs(updateResult.err, &wfNotReady) - s.Contains(updateResult.err.Error(), "Unable to perform workflow execution update due to unexpected workflow task failure.") + // Update is aborted, speculative WFT failure is recorded into the history. + updateResult := <-updateResultCh + var wfNotReady *serviceerror.WorkflowNotReady + s.ErrorAs(updateResult.err, &wfNotReady) + s.Contains(updateResult.err.Error(), "Unable to perform workflow execution update due to unexpected workflow task failure.") - // New transient WFT is created, but it is not shown in the history. - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + // New transient WFT is created, but it is not shown in the history. + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2193,13 +2211,13 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 6 WorkflowTaskStarted 7 WorkflowTaskFailed`, events) - // Send Update again. It will be delivered on existing transient WFT. - updateResultCh = sendUpdate(timeoutCtx, s, tv) + // Send Update again. It will be delivered on existing transient WFT. + updateResultCh = sendUpdate(timeoutCtx, s, s.Tv()) - // Try to accept 2nd update in workflow: get error. Poller will fail WFT, but the registry won't be cleared and Update won't be aborted. - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Try to accept 2nd update in workflow: get error. Poller will fail WFT, but the registry won't be cleared and Update won't be aborted. + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2210,23 +2228,24 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 8 WorkflowTaskScheduled // Transient WFT 9 WorkflowTaskStarted`, task.History) - updRequestMsg := task.Messages[0] - s.EqualValues(8, updRequestMsg.GetEventId()) - // Returning error will cause the poller to fail WFT. - return nil, errors.New("malformed request") - }) - // The error is from RespondWorkflowTaskFailed, which should go w/o error. - s.NoError(err) + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] + s.EqualValues(8, updRequestMsg.GetEventId()) + // Returning error will cause the poller to fail WFT. + return nil, errors.New("malformed request") + }) + // The error is from RespondWorkflowTaskFailed, which should go w/o error. + s.NoError(err) - // Update timed out, but stays in the registry and will be delivered again on the new transient WFT. - updateResult = <-updateResultCh - s.Error(updateResult.err) - s.True(common.IsContextDeadlineExceededErr(updateResult.err), "UpdateWorkflowExecution must timeout after 2 seconds") - s.Nil(updateResult.response) + // Update timed out, but stays in the registry and will be delivered again on the new transient WFT. + updateResult = <-updateResultCh + s.Error(updateResult.err) + s.True(common.IsContextDeadlineExceededErr(updateResult.err), "UpdateWorkflowExecution must timeout after 2 seconds") + s.Nil(updateResult.response) - // This WFT failure wasn't recorded because WFT was transient. - events = s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + // This WFT failure wasn't recorded because WFT was transient. + events = s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2235,22 +2254,23 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 6 WorkflowTaskStarted 7 WorkflowTaskFailed`, events) - // Try to accept 2nd update in workflow 2nd time: get error. Poller will fail WT. Update is not aborted. - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - // 1st attempt UpdateWorkflowExecution call has timed out but the - // update is still running - updRequestMsg := task.Messages[0] - s.EqualValues(8, updRequestMsg.GetEventId()) - // Fail WT one more time. This is transient WT and shouldn't appear in the history. - // Returning error will cause the poller to fail WT. - return nil, errors.New("malformed request") - }) - // The error is from RespondWorkflowTaskFailed, which should go w/o error. - s.NoError(err) + // Try to accept 2nd update in workflow 2nd time: get error. Poller will fail WT. Update is not aborted. + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + // 1st attempt UpdateWorkflowExecution call has timed out but the + // update is still running + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] + s.EqualValues(8, updRequestMsg.GetEventId()) + // Fail WT one more time. This is transient WT and shouldn't appear in the history. + // Returning error will cause the poller to fail WT. + return nil, errors.New("malformed request") + }) + // The error is from RespondWorkflowTaskFailed, which should go w/o error. + s.NoError(err) - events = s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events = s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2259,10 +2279,10 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 6 WorkflowTaskStarted 7 WorkflowTaskFailed`, events) - // Complete Update and workflow. - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` + // Complete Update and workflow. + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2273,18 +2293,19 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 8 WorkflowTaskScheduled // Transient WFT 9 WorkflowTaskStarted`, task.History) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - Commands: append(s.UpdateAcceptCompleteCommands(tv), &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }), - }, nil - }) - s.NoError(err) + s.Require().NotEmpty(task.Messages, "expected update message in task") + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + Commands: append(s.UpdateAcceptCompleteCommands(s.Tv()), &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }), + }, nil + }) + s.NoError(err) - events = s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events = s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2298,22 +2319,21 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Fail() { 11 WorkflowExecutionUpdateAccepted 12 WorkflowExecutionUpdateCompleted 13 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_ConvertToNormalBecauseOfBufferedSignal() { - tv := testvars.New(s.T()) + t.Run("StartedSpeculativeWorkflowTask_ConvertToNormalBecauseOfBufferedSignal", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2321,85 +2341,84 @@ func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_ConvertToNormal 5 WorkflowTaskScheduled // Speculative WT. Events 5 and 6 are written into the history when signal is received. 6 WorkflowTaskStarted `, task.History) - // Send signal which will be buffered. This will persist MS and speculative WT must be converted to normal. - err := s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - return nil, nil - case 3: - s.EqualHistory(` + // Send signal which will be buffered. This will persist MS and speculative WT must be converted to normal. + err := s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) + return nil, nil + case 3: + s.EqualHistory(` 7 WorkflowTaskCompleted 8 WorkflowExecutionSignaled // It was buffered and got to the history after WT is completed. 9 WorkflowTaskScheduled 10 WorkflowTaskStarted`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1, 3: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] - s.EqualValues(5, updRequestMsg.GetEventId()) + s.EqualValues(5, updRequestMsg.GetEventId()) - // Update is rejected but corresponding speculative WT will be in the history anyway, because it was converted to normal due to buffered signal. - return s.UpdateRejectMessages(tv, updRequestMsg), nil - case 3: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + // Update is rejected but corresponding speculative WT will be in the history anyway, because it was converted to normal due to buffered signal. + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, updateResp.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, updateResp.ResetHistoryEventId) - // Complete workflow. - completeWorkflowResp, err := poller.HandlePartialWorkflowTask(updateResp.GetWorkflowTask(), false) - s.NoError(err) - s.NotNil(completeWorkflowResp) - s.Nil(completeWorkflowResp.GetWorkflowTask()) - s.EqualValues(0, completeWorkflowResp.ResetHistoryEventId) + // Complete workflow. + completeWorkflowResp, err := poller.HandlePartialWorkflowTask(updateResp.GetWorkflowTask(), false) + s.NoError(err) + s.NotNil(completeWorkflowResp) + s.Nil(completeWorkflowResp.GetWorkflowTask()) + s.EqualValues(0, completeWorkflowResp.ResetHistoryEventId) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2412,22 +2431,21 @@ func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_ConvertToNormal 10 WorkflowTaskStarted 11 WorkflowTaskCompleted 12 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_ConvertToNormalBecauseOfSignal() { - tv := testvars.New(s.T()) + t.Run("ScheduledSpeculativeWorkflowTask_ConvertToNormalBecauseOfSignal", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2435,69 +2453,71 @@ func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_ConvertToNorm 5 WorkflowTaskScheduled // It was initially speculative WT but was already converted to normal when signal was received. 6 WorkflowExecutionSignaled 7 WorkflowTaskStarted`, task.History) - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Require().NotEmpty(task.Messages, "expected update message in task") + updRequestMsg := task.Messages[0] - s.EqualValues(6, updRequestMsg.GetEventId()) + s.EqualValues(6, updRequestMsg.GetEventId()) - // Update is rejected but corresponding speculative WT was already converted to normal, - // and will be in the history anyway. - return s.UpdateRejectMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + // Update is rejected but corresponding speculative WT was already converted to normal, + // and will be in the history anyway. + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Send signal which will NOT be buffered because speculative WT is not started yet (only scheduled). - // This will persist MS and speculative WT must be converted to normal. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) + // Send signal which will NOT be buffered because speculative WT is not started yet (only scheduled). + // This will persist MS and speculative WT must be converted to normal. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2507,35 +2527,35 @@ func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_ConvertToNorm 7 WorkflowTaskStarted 8 WorkflowTaskCompleted // Update was rejected but WT events 5,7,8 are in the history because of signal. `, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_StartToCloseTimeout() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_StartToCloseTimeout", func(t *testing.T) { + // Uses CaptureMetricsHandler which requires a dedicated cluster to avoid metric interference. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() - defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) + request := &workflowservice.StartWorkflowExecutionRequest{ + RequestId: s.Tv().Any().String(), + Namespace: s.Namespace().String(), + WorkflowId: s.Tv().WorkflowID(), + WorkflowType: s.Tv().WorkflowType(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskTimeout: durationpb.New(1 * time.Second), // Important! + } - request := &workflowservice.StartWorkflowExecutionRequest{ - RequestId: tv.Any().String(), - Namespace: s.Namespace().String(), - WorkflowId: tv.WorkflowID(), - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskTimeout: durationpb.New(1 * time.Second), // Important! - } + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), request) + s.NoError(err) - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), request) - s.NoError(err) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2543,13 +2563,13 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_StartToCloseTimeout() 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted `, task.History) - // Emulate slow worker: sleep little more than WT timeout. - time.Sleep(request.WorkflowTaskTimeout.AsDuration() + 100*time.Millisecond) //nolint:forbidigo - // This doesn't matter because WT times out before update is applied. - return s.UpdateAcceptCompleteCommands(tv), nil - case 3: - // Speculative WT timed out and retried as normal WT. - s.EqualHistory(` + // Emulate slow worker: sleep little more than WT timeout. + time.Sleep(request.WorkflowTaskTimeout.AsDuration() + 100*time.Millisecond) //nolint:forbidigo + // This doesn't matter because WT times out before update is applied. + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + case 3: + // Speculative WT timed out and retried as normal WT. + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2559,98 +2579,99 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_StartToCloseTimeout() 7 WorkflowTaskTimedOut 8 WorkflowTaskScheduled {"Attempt":2 } // Transient WT. 9 WorkflowTaskStarted`, task.History) - commands := append(s.UpdateAcceptCompleteCommands(tv), - &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }) - return commands, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + commands := append(s.UpdateAcceptCompleteCommands(s.Tv()), + &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }) + return commands, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1) - updRequestMsg := task.Messages[0] + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Len(task.Messages, 1) + updRequestMsg := task.Messages[0] - // This doesn't matter because WT times out before update is applied. - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - case 3: - // Update is still in registry and was sent again. - updRequestMsg := task.Messages[0] + // This doesn't matter because WT times out before update is applied. + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + case 3: + // Update is still in registry and was sent again. + updRequestMsg := task.Messages[0] - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err = poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, tv) - - // Try to process update in workflow, but it takes more than WT timeout. So, WT times out. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.Error(err) - s.Equal("Workflow task not found.", err.Error()) - - // ensure correct metrics were recorded - snap := capture.Snapshot() - - var speculativeWorkflowTaskTimeoutTasks int - for _, m := range snap[metrics.TaskRequests.Name()] { - if m.Tags[metrics.OperationTagName] == metrics.TaskTypeTimerActiveTaskSpeculativeWorkflowTaskTimeout { - speculativeWorkflowTaskTimeoutTasks += 1 + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), } - } - s.Equal(1, speculativeWorkflowTaskTimeoutTasks, "expected 1 speculative workflow task timeout task to be created") - var speculativeStartToCloseTimeouts int - for _, m := range snap[metrics.StartToCloseTimeoutCounter.Name()] { - if m.Tags[metrics.OperationTagName] == metrics.TaskTypeTimerActiveTaskSpeculativeWorkflowTaskTimeout { - speculativeStartToCloseTimeouts += 1 + // Drain first WT. + _, err = poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Try to process update in workflow, but it takes more than WT timeout. So, WT times out. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.Error(err) + s.Equal("Workflow task not found.", err.Error()) + + // ensure correct metrics were recorded + snap := capture.Snapshot() + + var speculativeWorkflowTaskTimeoutTasks int + for _, m := range snap[metrics.TaskRequests.Name()] { + if m.Tags[metrics.OperationTagName] == metrics.TaskTypeTimerActiveTaskSpeculativeWorkflowTaskTimeout { + speculativeWorkflowTaskTimeoutTasks += 1 + } } - } - s.Equal(1, speculativeStartToCloseTimeouts, "expected 1 timeout of a speculative workflow task timeout task") + s.Equal(1, speculativeWorkflowTaskTimeoutTasks, "expected 1 speculative workflow task timeout task to be created") - // New normal WT was created on server after speculative WT has timed out. - // It will accept and complete update first and workflow itself with the same WT. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, updateResp.ResetHistoryEventId) - s.Nil(updateResp.GetWorkflowTask()) + var speculativeStartToCloseTimeouts int + for _, m := range snap[metrics.StartToCloseTimeoutCounter.Name()] { + if m.Tags[metrics.OperationTagName] == metrics.TaskTypeTimerActiveTaskSpeculativeWorkflowTaskTimeout { + speculativeStartToCloseTimeouts += 1 + } + } + s.Equal(1, speculativeStartToCloseTimeouts, "expected 1 timeout of a speculative workflow task timeout task") - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + // New normal WT was created on server after speculative WT has timed out. + // It will accept and complete update first and workflow itself with the same WT. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, updateResp.ResetHistoryEventId) + s.Nil(updateResp.GetWorkflowTask()) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2662,36 +2683,35 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_StartToCloseTimeout() 9 WorkflowTaskStarted 10 WorkflowTaskCompleted 11 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 8} // WTScheduled event which delivered update to the worker. - 12 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 11} + 12 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 11} 13 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_ScheduleToStartTimeout", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + // Drain first WT and respond with sticky enabled response to enable sticky task queue. + stickyScheduleToStartTimeout := 1 * time.Second + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + StickyAttributes: s.Tv().StickyExecutionAttributes(stickyScheduleToStartTimeout), + }, nil + }) + s.NoError(err) - // Drain first WT and respond with sticky enabled response to enable sticky task queue. - stickyScheduleToStartTimeout := 1 * time.Second - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - StickyAttributes: tv.StickyExecutionAttributes(stickyScheduleToStartTimeout), - }, nil - }) - s.NoError(err) + sendUpdateNoError(s, s.Tv()) - sendUpdateNoError(s, tv) + s.Logger.Info("Wait for sticky timeout to fire. Sleep poller.StickyScheduleToStartTimeout+ seconds.", tag.NewDurationTag("StickyScheduleToStartTimeout", stickyScheduleToStartTimeout)) + time.Sleep(stickyScheduleToStartTimeout + 100*time.Millisecond) //nolint:forbidigo + s.Logger.Info("Sleep is done.") - s.Logger.Info("Wait for sticky timeout to fire. Sleep poller.StickyScheduleToStartTimeout+ seconds.", tag.NewDurationTag("StickyScheduleToStartTimeout", stickyScheduleToStartTimeout)) - time.Sleep(stickyScheduleToStartTimeout + 100*time.Millisecond) //nolint:forbidigo - s.Logger.Info("Sleep is done.") - - // Try to process update in workflow, poll from normal task queue. - res, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - // Speculative WFT timed out on sticky task queue. Server sent full history with sticky timeout event. - s.EqualHistory(` + // Try to process update in workflow, poll from normal task queue. + res, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + // Speculative WFT timed out on sticky task queue. Server sent full history with sticky timeout event. + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2701,16 +2721,16 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout 7 WorkflowTaskScheduled {"Attempt":1} // Normal WT. 8 WorkflowTaskStarted`, task.History) - // Reject update, but WFT will still be in the history due to timeout on sticky queue. - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - s.NotNil(res) + // Reject update, but WFT will still be in the history due to timeout on sticky queue. + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + s.NotNil(res) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2721,22 +2741,21 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout 8 WorkflowTaskStarted 9 WorkflowTaskCompleted // Normal WT is completed and events are in the history even update was rejected. `, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeoutOnNormalTaskQueue() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_ScheduleToStartTimeoutOnNormalTaskQueue", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2746,73 +2765,74 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout 7 WorkflowTaskScheduled {"Attempt":1} // Normal WT is scheduled. 8 WorkflowTaskStarted `, task.History) - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(7, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(7, updRequestMsg.GetEventId()) - return s.UpdateRejectMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateRejectMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain existing WT from normal task queue. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) + // Drain existing WT from normal task queue. + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) - // Now send an update. It will create a speculative WT on normal task queue, - // which will time out in 5 seconds and create new normal WT. - updateResultCh := sendUpdateNoError(s, tv) + // Now send an update. It will create a speculative WT on normal task queue, + // which will time out in 5 seconds and create new normal WT. + updateResultCh := sendUpdateNoError(s, s.Tv()) - // TODO: it would be nice to shutdown matching before sending an update to emulate case which is actually being tested here. - // But test infrastructure doesn't support it. 5 seconds sleep will cause same observable effect. - s.Logger.Info("Sleep 5+ seconds to make sure tasks.SpeculativeWorkflowTaskScheduleToStartTimeout time has passed.") - time.Sleep(5*time.Second + 100*time.Millisecond) //nolint:forbidigo - s.Logger.Info("Sleep 5+ seconds is done.") + // TODO: it would be nice to shutdown matching before sending an update to emulate case which is actually being tested here. + // But test infrastructure doesn't support it. 5 seconds sleep will cause same observable effect. + s.Logger.Info("Sleep 5+ seconds to make sure tasks.SpeculativeWorkflowTaskScheduleToStartTimeout time has passed.") + time.Sleep(5*time.Second + 100*time.Millisecond) //nolint:forbidigo + s.Logger.Info("Sleep 5+ seconds is done.") - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with normal workflow task") + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, res.NewTask.ResetHistoryEventId, "no reset of event ID should happened after update rejection if it was delivered with normal workflow task") - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2823,96 +2843,96 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_ScheduleToStartTimeout 8 WorkflowTaskStarted 9 WorkflowTaskCompleted `, events) -} + }) -func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_TerminateWorkflow() { - tv := testvars.New(s.T()) + t.Run("StartedSpeculativeWorkflowTask_TerminateWorkflow", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + // Terminate workflow while speculative WT is running. + _, err := s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) + s.NoError(err) - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - // Terminate workflow while speculative WT is running. - _, err := s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) - - s.EqualHistory(` + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - oneSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 1*time.Second) - defer cancel() - updateResultCh := sendUpdate(oneSecondTimeoutCtx, s, tv) + oneSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 1*time.Second) + defer cancel() + updateResultCh := sendUpdate(oneSecondTimeoutCtx, s, s.Tv()) - // Process update in workflow. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.Error(err) - s.IsType(err, (*serviceerror.NotFound)(nil)) - s.ErrorContains(err, "Workflow task not found.") + // Process update in workflow. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.Error(err) + s.ErrorAs(err, new(*serviceerror.NotFound)) + s.ErrorContains(err, "Workflow task not found.") - updateResult := <-updateResultCh - s.Error(updateResult.err) - var notFound *serviceerror.NotFound - s.ErrorAs(updateResult.err, ¬Found) - s.ErrorContains(updateResult.err, update.AbortedByWorkflowClosingErr.Error()) - s.Nil(updateResult.response) + updateResult := <-updateResultCh + s.Error(updateResult.err) + var notFound *serviceerror.NotFound + s.ErrorAs(updateResult.err, ¬Found) + s.ErrorContains(updateResult.err, update.AbortedByWorkflowClosingErr.Error()) + s.Nil(updateResult.response) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -2922,85 +2942,85 @@ func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_TerminateWorkfl 7 WorkflowTaskFailed 8 WorkflowExecutionTerminated`, events) - msResp, err := s.AdminClient().DescribeMutableState(testcore.NewContext(), &adminservice.DescribeMutableStateRequest{ - Namespace: s.Namespace().String(), - Execution: tv.WorkflowExecution(), - Archetype: chasm.WorkflowArchetype, + msResp, err := s.AdminClient().DescribeMutableState(testcore.NewContext(), &adminservice.DescribeMutableStateRequest{ + Namespace: s.Namespace().String(), + Execution: s.Tv().WorkflowExecution(), + Archetype: chasm.WorkflowArchetype, + }) + s.NoError(err) + s.EqualValues(7, msResp.GetDatabaseMutableState().GetExecutionInfo().GetCompletionEventBatchId(), "completion_event_batch_id should point to WTFailed event") }) - s.NoError(err) - s.EqualValues(7, msResp.GetDatabaseMutableState().GetExecutionInfo().GetCompletionEventBatchId(), "completion_event_batch_id should point to WTFailed event") -} -func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_TerminateWorkflow() { - tv := testvars.New(s.T()) + t.Run("ScheduledSpeculativeWorkflowTask_TerminateWorkflow", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - oneSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 1*time.Second) - defer cancel() - updateResultCh := sendUpdate(oneSecondTimeoutCtx, s, tv) + oneSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 1*time.Second) + defer cancel() + updateResultCh := sendUpdate(oneSecondTimeoutCtx, s, s.Tv()) - // Terminate workflow after speculative WT is scheduled but not started. - _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) + // Terminate workflow after speculative WT is scheduled but not started. + _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) + s.NoError(err) - updateResult := <-updateResultCh - s.Error(updateResult.err) - var notFound *serviceerror.NotFound - s.ErrorAs(updateResult.err, ¬Found) - s.ErrorContains(updateResult.err, update.AbortedByWorkflowClosingErr.Error()) - s.Nil(updateResult.response) + updateResult := <-updateResultCh + s.Error(updateResult.err) + var notFound *serviceerror.NotFound + s.ErrorAs(updateResult.err, ¬Found) + s.ErrorContains(updateResult.err, update.AbortedByWorkflowClosingErr.Error()) + s.Nil(updateResult.response) - s.Equal(1, wtHandlerCalls) - s.Equal(1, msgHandlerCalls) + s.Equal(1, wtHandlerCalls) + s.Equal(1, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -3008,137 +3028,860 @@ func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_TerminateWork 5 WorkflowExecutionTerminated // Speculative WTScheduled event is not written to history if WF is terminated. `, events) - msResp, err := s.AdminClient().DescribeMutableState(testcore.NewContext(), &adminservice.DescribeMutableStateRequest{ - Namespace: s.Namespace().String(), - Execution: tv.WorkflowExecution(), - Archetype: chasm.WorkflowArchetype, + msResp, err := s.AdminClient().DescribeMutableState(testcore.NewContext(), &adminservice.DescribeMutableStateRequest{ + Namespace: s.Namespace().String(), + Execution: s.Tv().WorkflowExecution(), + Archetype: chasm.WorkflowArchetype, + }) + s.NoError(err) + s.EqualValues(5, msResp.GetDatabaseMutableState().GetExecutionInfo().GetCompletionEventBatchId(), "completion_event_batch_id should point to WFTerminated event") }) - s.NoError(err) - s.EqualValues(5, msResp.GetDatabaseMutableState().GetExecutionInfo().GetCompletionEventBatchId(), "completion_event_batch_id should point to WFTerminated event") -} -func (s *UpdateWorkflowSuite) TestCompleteWorkflow_AbortUpdates() { - type testCase struct { - name string - description string - updateErr map[string]string // Update error by completionCommand.Name. - updateFailure string - commands func(tv *testvars.TestVars) []*commandpb.Command - messages func(tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message - } - type completionCommand struct { - name string - finalStatus enumspb.WorkflowExecutionStatus - useRunID bool - command func(tv *testvars.TestVars) *commandpb.Command - } - testCases := []testCase{ - { - name: "update admitted", - description: "update in stateAdmitted must get an error", - updateErr: map[string]string{ - "workflow completed": update.AbortedByWorkflowClosingErr.Error(), - "workflow continued as new without runID": "workflow operation can not be applied because workflow is closing", - "workflow continued as new with runID": "workflow operation can not be applied because workflow is closing", - "workflow failed": update.AbortedByWorkflowClosingErr.Error(), + t.Run("CompleteWorkflow_AbortUpdates", func(t *testing.T) { + type testCase struct { + name string + description string + updateErr map[string]string // Update error by completionCommand.Name. + updateFailure string + commands func(s *testcore.FunctionalTestBase, tv *testvars.TestVars) []*commandpb.Command + messages func(s *testcore.FunctionalTestBase, tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message + } + type completionCommand struct { + name string + finalStatus enumspb.WorkflowExecutionStatus + useRunID bool + command func(tv *testvars.TestVars) *commandpb.Command + } + testCases := []testCase{ + { + name: "update admitted", + description: "update in stateAdmitted must get an error", + updateErr: map[string]string{ + "workflow completed": update.AbortedByWorkflowClosingErr.Error(), + "workflow continued as new without runID": "workflow operation can not be applied because workflow is closing", + "workflow continued as new with runID": "workflow operation can not be applied because workflow is closing", + "workflow failed": update.AbortedByWorkflowClosingErr.Error(), + }, + updateFailure: "", + commands: func(s *testcore.FunctionalTestBase, _ *testvars.TestVars) []*commandpb.Command { return nil }, + messages: func(s *testcore.FunctionalTestBase, _ *testvars.TestVars, _ *protocolpb.Message) []*protocolpb.Message { + return nil + }, }, - updateFailure: "", - commands: func(_ *testvars.TestVars) []*commandpb.Command { return nil }, - messages: func(_ *testvars.TestVars, _ *protocolpb.Message) []*protocolpb.Message { return nil }, - }, - { - name: "update accepted", - description: "update in stateAccepted must get an update failure", - updateErr: map[string]string{"*": ""}, - updateFailure: "Workflow Update failed because the Workflow completed before the Update completed.", - commands: func(tv *testvars.TestVars) []*commandpb.Command { return s.UpdateAcceptCommands(tv) }, - messages: func(tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { - return s.UpdateAcceptMessages(tv, updRequestMsg) + { + name: "update accepted", + description: "update in stateAccepted must get an update failure", + updateErr: map[string]string{"*": ""}, + updateFailure: "Workflow Update failed because the Workflow completed before the Update completed.", + commands: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars) []*commandpb.Command { + return s.UpdateAcceptCommands(tv) + }, + messages: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { + return s.UpdateAcceptMessages(tv, updRequestMsg) + }, }, - }, - { - name: "update completed", - description: "completed update must not be affected by workflow completion", - updateErr: map[string]string{"*": ""}, - updateFailure: "", - commands: func(tv *testvars.TestVars) []*commandpb.Command { return s.UpdateAcceptCompleteCommands(tv) }, - messages: func(tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg) + { + name: "update completed", + description: "completed update must not be affected by workflow completion", + updateErr: map[string]string{"*": ""}, + updateFailure: "", + commands: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars) []*commandpb.Command { + return s.UpdateAcceptCompleteCommands(tv) + }, + messages: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { + return s.UpdateAcceptCompleteMessages(tv, updRequestMsg) + }, }, - }, - { - name: "update rejected", - description: "rejected update must be rejected with rejection from workflow", - updateErr: map[string]string{"*": ""}, - updateFailure: "rejection-of-", // Rejection from workflow. - commands: func(tv *testvars.TestVars) []*commandpb.Command { return nil }, - messages: func(tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { - return s.UpdateRejectMessages(tv, updRequestMsg) + { + name: "update rejected", + description: "rejected update must be rejected with rejection from workflow", + updateErr: map[string]string{"*": ""}, + updateFailure: "rejection-of-", // Rejection from workflow. + commands: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars) []*commandpb.Command { return nil }, + messages: func(s *testcore.FunctionalTestBase, tv *testvars.TestVars, updRequestMsg *protocolpb.Message) []*protocolpb.Message { + return s.UpdateRejectMessages(tv, updRequestMsg) + }, }, - }, - } + } - workflowCompletionCommands := []completionCommand{ - { - name: "workflow completed", - finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - useRunID: false, - command: func(_ *testvars.TestVars) *commandpb.Command { - return &commandpb.Command{ + workflowCompletionCommands := []completionCommand{ + { + name: "workflow completed", + finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + useRunID: false, + command: func(_ *testvars.TestVars) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + } + }, + }, + { + name: "workflow continued as new with runID", + finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_CONTINUED_AS_NEW, + useRunID: true, + command: func(tv *testvars.TestVars) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + }}, + } + }, + }, + { + name: "workflow continued as new without runID", + finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, // This is the status of new run because update doesn't go to particular runID. + useRunID: false, + command: func(tv *testvars.TestVars) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + }}, + } + }, + }, + { + name: "workflow failed", + finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_FAILED, + useRunID: true, + command: func(tv *testvars.TestVars) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_FailWorkflowExecutionCommandAttributes{FailWorkflowExecutionCommandAttributes: &commandpb.FailWorkflowExecutionCommandAttributes{ + Failure: tv.Any().ApplicationFailure(), + }}, + } + }, + }, + } + + for _, tc := range testCases { + for _, wfCC := range workflowCompletionCommands { + t.Run(tc.name+" "+wfCC.name, func(t *testing.T) { + s := testcore.NewEnv(t) + runID := mustStartWorkflow(s, s.Tv()) + tv := s.Tv() + if wfCC.useRunID { + tv = tv.WithRunID(runID) + } + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + return append(tc.commands(s.FunctionalTestBase, s.Tv()), wfCC.command(s.Tv())), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + return tc.messages(s.FunctionalTestBase, s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResultCh := sendUpdate(testcore.NewContext(), s, tv) + + // Complete workflow. + _, err = poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResult := <-updateResultCh + expectedUpdateErr := tc.updateErr[wfCC.name] + if expectedUpdateErr == "" { + expectedUpdateErr = tc.updateErr["*"] + } + if expectedUpdateErr != "" { + s.Error(updateResult.err, tc.description) + s.Equal(expectedUpdateErr, updateResult.err.Error()) + } else { + s.NoError(updateResult.err, tc.description) + } + + if tc.updateFailure != "" { + s.NotNil(updateResult.response.GetOutcome().GetFailure(), tc.description) + s.Contains(updateResult.response.GetOutcome().GetFailure().GetMessage(), tc.updateFailure, tc.description) + } else { + s.Nil(updateResult.response.GetOutcome().GetFailure(), tc.description) + } + + if expectedUpdateErr == "" && tc.updateFailure == "" { + s.Equal(runID, updateResult.response.GetUpdateRef().GetWorkflowExecution().GetRunId(), "update wasn't applied to the same run as was started") + } + + // Check that update didn't block workflow completion. + descResp, err := s.FrontendClient().DescribeWorkflowExecution(testcore.NewContext(), &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + Execution: tv.WorkflowExecution(), + }) + s.NoError(err) + s.Equal(wfCC.finalStatus, descResp.WorkflowExecutionInfo.Status) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + }) + } + } + }) + + t.Run("SpeculativeWorkflowTask_Heartbeat", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + // Drain first WT. + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Heartbeat from speculative WT (no messages, no commands). + var updRequestMsg *protocolpb.Message + res, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Events (5 and 6) are for speculative WT, but they won't disappear after reject because speculative WT is converted to normal during heartbeat. + 6 WorkflowTaskStarted + `, task.History) + + s.Len(task.Messages, 1) + updRequestMsg = task.Messages[0] + s.EqualValues(5, updRequestMsg.GetEventId()) + + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + ReturnNewWorkflowTask: true, + ForceCreateNewWorkflowTask: true, + }, nil + }) + s.NoError(err) + + // Reject update from workflow. + updateResp, err := s.TaskPoller().HandleWorkflowTask(s.Tv(), + res.GetWorkflowTask(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistory(` + 7 WorkflowTaskCompleted + 8 WorkflowTaskScheduled // New WT (after heartbeat) is normal and won't disappear from the history after reject. + 9 WorkflowTaskStarted + `, task.History) + + s.Empty(task.Messages) + + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), updRequestMsg), + }, nil + }) + s.NoError(err) + s.NotNil(updateResp) + + updateResult := <-updateResultCh + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(0, updateResp.ResetHistoryEventId, "no reset of event ID should happened after update rejection because of heartbeat") + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled + 6 WorkflowTaskStarted + 7 WorkflowTaskCompleted // Heartbeat response. + 8 WorkflowTaskScheduled + 9 WorkflowTaskStarted + 10 WorkflowTaskCompleted // After heartbeat new normal WT was created and events are written into the history even update is rejected. + `, events) + }) + + t.Run("ScheduledSpeculativeWorkflowTask_LostUpdate", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionSignaled + 6 WorkflowTaskScheduled + 7 WorkflowTaskStarted`, task.History) + return []*commandpb.Command{{ CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - } - }, - }, - { - name: "workflow continued as new with runID", - finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_CONTINUED_AS_NEW, - useRunID: true, - command: func(tv *testvars.TestVars) *commandpb.Command { - return &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), - }}, - } - }, - }, - { - name: "workflow continued as new without runID", - finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, // This is the status of new run because update doesn't go to particular runID. - useRunID: false, - command: func(tv *testvars.TestVars) *commandpb.Command { - return &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), - }}, - } - }, - }, - { - name: "workflow failed", - finalStatus: enumspb.WORKFLOW_EXECUTION_STATUS_FAILED, - useRunID: true, - command: func(tv *testvars.TestVars) *commandpb.Command { - return &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_FailWorkflowExecutionCommandAttributes{FailWorkflowExecutionCommandAttributes: &commandpb.FailWorkflowExecutionCommandAttributes{ - Failure: tv.Any().ApplicationFailure(), - }}, - } - }, - }, - } + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } - for _, tc := range testCases { - for _, wfCC := range workflowCompletionCommands { - s.Run(tc.name+" "+wfCC.name, func() { - tv := testvars.New(s.T()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Empty(task.Messages, "update lost due to lost update registry") + return nil, nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } - runID := mustStartWorkflow(s, tv) + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) + defer cancel() + updateResult := <-sendUpdate(halfSecondTimeoutCtx, s, s.Tv()) + s.Error(updateResult.err) + s.True(common.IsContextDeadlineExceededErr(updateResult.err), updateResult.err.Error()) + s.Nil(updateResult.response) + + // Lose update registry. Speculative WFT and update registry disappear. + loseUpdateRegistryAndAbandonPendingUpdates(s, s.Tv()) + + // Ensure, there is no WFT. + pollCtx, cancel := context.WithTimeout(testcore.NewContext(), common.MinLongPollTimeout*2) + defer cancel() + pollResponse, err := s.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + }) + s.NoError(err) + s.Nil(pollResponse.Messages, "there should not be new WFT with messages") + + // Send signal to schedule new WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) + + // Complete workflow and check that there is update messages. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + s.NotNil(completeWorkflowResp) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionSignaled + 6 WorkflowTaskScheduled + 7 WorkflowTaskStarted + 8 WorkflowTaskCompleted + 9 WorkflowExecutionCompleted`, events) + }) + + t.Run("StartedSpeculativeWorkflowTask_LostUpdate", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Speculative WT. Events 5 and 6 will be lost. + 6 WorkflowTaskStarted +`, task.History) + + // Lose update registry. Update is lost and NotFound error will be returned to RespondWorkflowTaskCompleted. + loseUpdateRegistryAndAbandonPendingUpdates(s, s.Tv()) + + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + case 3: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionSignaled + 6 WorkflowTaskScheduled + 7 WorkflowTaskStarted +`, task.History) + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + s.EqualValues(5, updRequestMsg.GetEventId()) + + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + case 3: + s.Empty(task.Messages, "no messages since update registry was lost") + return nil, nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) + defer cancel() + updateResultCh := sendUpdate(halfSecondTimeoutCtx, s, s.Tv()) + + // Process update in workflow. + _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.Error(err) + s.ErrorAs(err, new(*serviceerror.NotFound)) + s.ErrorContains(err, "Workflow task not found") + + updateResult := <-updateResultCh + s.Error(updateResult.err) + s.True(common.IsContextDeadlineExceededErr(updateResult.err), updateResult.err.Error()) + s.Nil(updateResult.response) + + // Send signal to schedule new WFT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) + + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + s.NotNil(completeWorkflowResp) + + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionSignaled + 6 WorkflowTaskScheduled + 7 WorkflowTaskStarted + 8 WorkflowTaskCompleted + 9 WorkflowExecutionCompleted`, events) + }) + + t.Run("FirstNormalWorkflowTask_UpdateResurrectedAfterRegistryCleared", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted +`, task.History) + // Clear update registry. Update will be resurrected in registry from acceptance message. + clearUpdateRegistryAndAbortPendingUpdates(s, s.Tv()) + + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + case 2: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionUpdateAccepted + 6 WorkflowExecutionUpdateCompleted + 7 WorkflowExecutionSignaled + 8 WorkflowTaskScheduled + 9 WorkflowTaskStarted +`, task.History) + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + updRequestMsg := task.Messages[0] + s.EqualValues(2, updRequestMsg.GetEventId()) + + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + case 2: + s.Empty(task.Messages, "update must be processed and not delivered again") + return nil, nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Process update in workflow. Update won't be found on server but will be resurrected from acceptance message and completed. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + + // Client receives resurrected Update outcome. + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + + // Signal to create new WFT which shouldn't get any updates. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) + + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(completeWorkflowResp) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowExecutionUpdateAccepted + 6 WorkflowExecutionUpdateCompleted + 7 WorkflowExecutionSignaled + 8 WorkflowTaskScheduled + 9 WorkflowTaskStarted + 10 WorkflowTaskCompleted + 11 WorkflowExecutionCompleted`, events) + }) + + t.Run("ScheduledSpeculativeWorkflowTask_DeduplicateID", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Speculative WT. + 6 WorkflowTaskStarted +`, task.History) + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Len(task.Messages, 1, "2nd update must be deduplicated by ID") + updRequestMsg := task.Messages[0] + + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Send second update with the same ID. + updateResultCh2 := sendUpdateNoError(s, s.Tv()) + + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + updateResult2 := <-updateResultCh2 + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) + s.EqualValues(0, updateResp.ResetHistoryEventId) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled + 6 WorkflowTaskStarted + 7 WorkflowTaskCompleted + 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. + 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} +`, events) + }) + + t.Run("StartedSpeculativeWorkflowTask_DeduplicateID", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + + var updateResultCh2 <-chan *workflowservice.UpdateWorkflowExecutionResponse + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + // Send second update with the same ID when WT is started but not completed. + updateResultCh2 = sendUpdateNoError(s, s.Tv()) + + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Speculative WT. + 6 WorkflowTaskStarted +`, task.History) + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + s.Len(task.Messages, 1, "2nd update should not has reached server yet") + updRequestMsg := task.Messages[0] + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } + + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + + updateResultCh := sendUpdateNoError(s, s.Tv()) + + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, res.NewTask.ResetHistoryEventId) + + updateResult2 := <-updateResultCh2 + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) + + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) + + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled + 6 WorkflowTaskStarted + 7 WorkflowTaskCompleted + 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. + 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} +`, events) + }) + + t.Run("CompletedSpeculativeWorkflowTask_DeduplicateID", func(t *testing.T) { + testCases := []struct { + Name string + CloseShard bool + }{ + { + Name: "no shard reload", + CloseShard: false, + }, + { + Name: "with shard reload", + CloseShard: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + // Uses closeShard conditionally which requires a dedicated cluster. + var opts []testcore.TestOption + if tc.CloseShard { + opts = append(opts, testcore.WithDedicatedCluster()) + } + s := testcore.NewEnv(t, opts...) + mustStartWorkflow(s, s.Tv()) wtHandlerCalls := 0 wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { @@ -3148,7 +3891,20 @@ func (s *UpdateWorkflowSuite) TestCompleteWorkflow_AbortUpdates() { // Completes first WT with empty command list. return nil, nil case 2: - return append(tc.commands(tv), wfCC.command(tv)), nil + s.EqualHistory(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled // Speculative WT. + 6 WorkflowTaskStarted +`, task.History) + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + case 3: + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }}, nil default: s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) return nil, nil @@ -3163,18 +3919,22 @@ func (s *UpdateWorkflowSuite) TestCompleteWorkflow_AbortUpdates() { return nil, nil case 2: updRequestMsg := task.Messages[0] - return tc.messages(tv, updRequestMsg), nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + case 3: + s.Empty(task.Messages, "2nd update must be deduplicated by ID ") + return nil, nil default: s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) return nil, nil } } + //nolint:staticcheck // SA1019 TaskPoller replacement needed poller := &testcore.TaskPoller{ Client: s.FrontendClient(), Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), WorkflowTaskHandler: wtHandler, MessageHandler: msgHandler, Logger: s.Logger, @@ -3185,779 +3945,53 @@ func (s *UpdateWorkflowSuite) TestCompleteWorkflow_AbortUpdates() { _, err := poller.PollAndProcessWorkflowTask() s.NoError(err) - updateResultCh := sendUpdate(testcore.NewContext(), s, s.useRunID(tv, wfCC.useRunID, runID)) + updateResultCh := sendUpdateNoError(s, s.Tv()) - // Complete workflow. + // Process update in workflow. _, err = poller.PollAndProcessWorkflowTask() s.NoError(err) - updateResult := <-updateResultCh - expectedUpdateErr := tc.updateErr[wfCC.name] - if expectedUpdateErr == "" { - expectedUpdateErr = tc.updateErr["*"] - } - if expectedUpdateErr != "" { - s.Error(updateResult.err, tc.description) - s.Equal(updateResult.err.Error(), expectedUpdateErr) - } else { - s.NoError(updateResult.err, tc.description) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + + if tc.CloseShard { + // Close shard to make sure that for completed updates deduplication works even after shard reload. + closeShard(s, s.Tv().WorkflowID()) } - if tc.updateFailure != "" { - s.NotNil(updateResult.response.GetOutcome().GetFailure(), tc.description) - s.Contains(updateResult.response.GetOutcome().GetFailure().GetMessage(), tc.updateFailure, tc.description) - } else { - s.Nil(updateResult.response.GetOutcome().GetFailure(), tc.description) - } + // Send second update with the same ID. It must return immediately. + updateResult2 := <-sendUpdateNoError(s, s.Tv()) - if expectedUpdateErr == "" && tc.updateFailure == "" { - s.Equal(runID, updateResult.response.GetUpdateRef().GetWorkflowExecution().GetRunId(), "update wasn't applied to the same run as was started") - } - - // Check that update didn't block workflow completion. - descResp, err := s.FrontendClient().DescribeWorkflowExecution(testcore.NewContext(), &workflowservice.DescribeWorkflowExecutionRequest{ + // Ensure, there is no new WT. + pollCtx, cancel := context.WithTimeout(testcore.NewContext(), common.MinLongPollTimeout*2) + defer cancel() + pollResponse, err := s.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ Namespace: s.Namespace().String(), - Execution: s.useRunID(tv, wfCC.useRunID, runID).WorkflowExecution(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), }) s.NoError(err) - s.Equal(wfCC.finalStatus, descResp.WorkflowExecutionInfo.Status) + s.Nil(pollResponse.Messages, "there must be no new WT") - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - }) - } - } -} + s.Equal( + "success-result-of-"+s.Tv().UpdateID(), + testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess()), + "results of the first update must be available") -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_Heartbeat() { - tv := testvars.New(s.T()) + // Send signal to schedule new WT. + err = s.SendSignal(s.Namespace().String(), s.Tv().WorkflowExecution(), s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.NoError(err) - mustStartWorkflow(s, tv) + // Complete workflow. + completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) + s.NotNil(completeWorkflowResp) - // Drain first WT. - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - updateResultCh := sendUpdateNoError(s, tv) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - // Heartbeat from speculative WT (no messages, no commands). - var updRequestMsg *protocolpb.Message - res, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Events (5 and 6) are for speculative WT, but they won't disappear after reject because speculative WT is converted to normal during heartbeat. - 6 WorkflowTaskStarted - `, task.History) - - s.Len(task.Messages, 1) - updRequestMsg = task.Messages[0] - s.EqualValues(5, updRequestMsg.GetEventId()) - - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - ReturnNewWorkflowTask: true, - ForceCreateNewWorkflowTask: true, - }, nil - }) - s.NoError(err) - - // Reject update from workflow. - updateResp, err := s.TaskPoller().HandleWorkflowTask(tv, - res.GetWorkflowTask(), - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistory(` - 7 WorkflowTaskCompleted - 8 WorkflowTaskScheduled // New WT (after heartbeat) is normal and won't disappear from the history after reject. - 9 WorkflowTaskStarted - `, task.History) - - s.Empty(task.Messages) - - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, updRequestMsg), - }, nil - }) - s.NoError(err) - s.NotNil(updateResp) - - updateResult := <-updateResultCh - s.Equal("rejection-of-"+tv.UpdateID(), updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(0, updateResp.ResetHistoryEventId, "no reset of event ID should happened after update rejection because of heartbeat") - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled - 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted // Heartbeat response. - 8 WorkflowTaskScheduled - 9 WorkflowTaskStarted - 10 WorkflowTaskCompleted // After heartbeat new normal WT was created and events are written into the history even update is rejected. - `, events) -} - -func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_LostUpdate() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionSignaled - 6 WorkflowTaskScheduled - 7 WorkflowTaskStarted`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Empty(task.Messages, "update lost due to lost update registry") - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) - defer cancel() - updateResult := <-sendUpdate(halfSecondTimeoutCtx, s, tv) - s.Error(updateResult.err) - s.True(common.IsContextDeadlineExceededErr(updateResult.err), updateResult.err.Error()) - s.Nil(updateResult.response) - - // Lose update registry. Speculative WFT and update registry disappear. - s.loseUpdateRegistryAndAbandonPendingUpdates(tv) - - // Ensure, there is no WFT. - pollCtx, cancel := context.WithTimeout(testcore.NewContext(), common.MinLongPollTimeout*2) - defer cancel() - pollResponse, err := s.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - }) - s.NoError(err) - s.Nil(pollResponse.Messages, "there should not be new WFT with messages") - - // Send signal to schedule new WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - - // Complete workflow and check that there is update messages. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - s.NotNil(completeWorkflowResp) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionSignaled - 6 WorkflowTaskScheduled - 7 WorkflowTaskStarted - 8 WorkflowTaskCompleted - 9 WorkflowExecutionCompleted`, events) -} - -func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_LostUpdate() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Speculative WT. Events 5 and 6 will be lost. - 6 WorkflowTaskStarted -`, task.History) - - // Lose update registry. Update is lost and NotFound error will be returned to RespondWorkflowTaskCompleted. - s.loseUpdateRegistryAndAbandonPendingUpdates(tv) - - return s.UpdateAcceptCompleteCommands(tv), nil - case 3: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionSignaled - 6 WorkflowTaskScheduled - 7 WorkflowTaskStarted -`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - s.EqualValues(5, updRequestMsg.GetEventId()) - - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - case 3: - s.Empty(task.Messages, "no messages since update registry was lost") - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - halfSecondTimeoutCtx, cancel := context.WithTimeout(testcore.NewContext(), 500*time.Millisecond) - defer cancel() - updateResultCh := sendUpdate(halfSecondTimeoutCtx, s, tv) - - // Process update in workflow. - _, err = poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.Error(err) - s.IsType(&serviceerror.NotFound{}, err) - s.ErrorContains(err, "Workflow task not found") - - updateResult := <-updateResultCh - s.Error(updateResult.err) - s.True(common.IsContextDeadlineExceededErr(updateResult.err), updateResult.err.Error()) - s.Nil(updateResult.response) - - // Send signal to schedule new WFT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - s.NotNil(completeWorkflowResp) - - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionSignaled - 6 WorkflowTaskScheduled - 7 WorkflowTaskStarted - 8 WorkflowTaskCompleted - 9 WorkflowExecutionCompleted`, events) -} - -func (s *UpdateWorkflowSuite) TestFirstNormalWorkflowTask_UpdateResurrectedAfterRegistryCleared() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted -`, task.History) - // Clear update registry. Update will be resurrected in registry from acceptance message. - s.clearUpdateRegistryAndAbortPendingUpdates(tv) - - return s.UpdateAcceptCompleteCommands(tv), nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionUpdateCompleted - 7 WorkflowExecutionSignaled - 8 WorkflowTaskScheduled - 9 WorkflowTaskStarted -`, task.History) - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - updRequestMsg := task.Messages[0] - s.EqualValues(2, updRequestMsg.GetEventId()) - - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - case 2: - s.Empty(task.Messages, "update must be processed and not delivered again") - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - updateResultCh := sendUpdateNoError(s, tv) - - // Process update in workflow. Update won't be found on server but will be resurrected from acceptance message and completed. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - - // Client receives resurrected Update outcome. - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - - // Signal to create new WFT which shouldn't get any updates. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(completeWorkflowResp) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionUpdateCompleted - 7 WorkflowExecutionSignaled - 8 WorkflowTaskScheduled - 9 WorkflowTaskStarted - 10 WorkflowTaskCompleted - 11 WorkflowExecutionCompleted`, events) -} - -func (s *UpdateWorkflowSuite) TestScheduledSpeculativeWorkflowTask_DeduplicateID() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Speculative WT. - 6 WorkflowTaskStarted -`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1, "2nd update must be deduplicated by ID") - updRequestMsg := task.Messages[0] - - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, tv) - - // Send second update with the same ID. - updateResultCh2 := sendUpdateNoError(s, tv) - - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - updateResult2 := <-updateResultCh2 - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) - s.EqualValues(0, updateResp.ResetHistoryEventId) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled - 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted - 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. - 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} -`, events) -} - -func (s *UpdateWorkflowSuite) TestStartedSpeculativeWorkflowTask_DeduplicateID() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - var updateResultCh2 <-chan *workflowservice.UpdateWorkflowExecutionResponse - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - // Send second update with the same ID when WT is started but not completed. - updateResultCh2 = sendUpdateNoError(s, tv) - - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Speculative WT. - 6 WorkflowTaskStarted -`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - s.Len(task.Messages, 1, "2nd update should not has reached server yet") - updRequestMsg := task.Messages[0] - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, tv) - - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) - - updateResult2 := <-updateResultCh2 - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess())) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled - 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted - 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. - 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} -`, events) -} - -func (s *UpdateWorkflowSuite) TestCompletedSpeculativeWorkflowTask_DeduplicateID() { - testCases := []struct { - Name string - CloseShard bool - }{ - { - Name: "no shard reload", - CloseShard: false, - }, - { - Name: "with shard reload", - CloseShard: true, - }, - } - - for _, tc := range testCases { - s.Run(tc.Name, func() { - tv := testvars.New(s.T()) - - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled // Speculative WT. - 6 WorkflowTaskStarted -`, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - case 3: - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - case 3: - s.Empty(task.Messages, "2nd update must be deduplicated by ID ") - return nil, nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } - - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - - updateResultCh := sendUpdateNoError(s, tv) - - // Process update in workflow. - _, err = poller.PollAndProcessWorkflowTask() - s.NoError(err) - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - - if tc.CloseShard { - // Close shard to make sure that for completed updates deduplication works even after shard reload. - s.closeShard(tv.WorkflowID()) - } - - // Send second update with the same ID. It must return immediately. - updateResult2 := <-sendUpdateNoError(s, tv) - - // Ensure, there is no new WT. - pollCtx, cancel := context.WithTimeout(testcore.NewContext(), common.MinLongPollTimeout*2) - defer cancel() - pollResponse, err := s.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - }) - s.NoError(err) - s.Nil(pollResponse.Messages, "there must be no new WT") - - s.EqualValues( - "success-result-of-"+tv.UpdateID(), - testcore.DecodeString(s.T(), updateResult2.GetOutcome().GetSuccess()), - "results of the first update must be available") - - // Send signal to schedule new WT. - err = s.SendSignal(s.Namespace().String(), tv.WorkflowExecution(), tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.NoError(err) - - // Complete workflow. - completeWorkflowResp, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) - s.NotNil(completeWorkflowResp) - - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) - - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -3973,79 +4007,80 @@ func (s *UpdateWorkflowSuite) TestCompletedSpeculativeWorkflowTask_DeduplicateID 13 WorkflowTaskCompleted 14 WorkflowExecutionCompleted `, events) - }) - } -} - -func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDifferentStartedId() { - /* - Test scenario: - An update created a speculative WT and WT is dispatched to the worker (started). - Update registry is cleared, speculative WT disappears from server. - Update is retired and second speculative WT is scheduled but not dispatched yet. - An activity completes, it converts the 2nd speculative WT into normal one. - The first speculative WT responds back, server fails request because WorkflowTaskStarted event Id is mismatched. - The second speculative WT responds back and server completes it. - */ - - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Schedule activity. - return []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: tv.ActivityID(), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: tv.Any().InfiniteTimeout(), - }}, - }}, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + }) } - } - - atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { - return tv.Any().Payloads(), false, nil - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - ActivityTaskHandler: atHandler, - Logger: s.Logger, - T: s.T(), - } - - // First WT will schedule activity. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - - // Send 1st update. It will create 2nd WT as speculative. - sendUpdateNoError(s, tv) - - // Poll 2nd speculative WT with 1st update. - wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), }) - s.NoError(err) - s.NotNil(wt2) - s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") - s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") - s.EqualValues(7, wt2.StartedEventId) - s.EqualValues(6, wt2.Messages[0].GetEventId()) - s.EqualHistory(` + + t.Run("StaleSpeculativeWorkflowTask_Fail_BecauseOfDifferentStartedId", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + /* + Test scenario: + An update created a speculative WT and WT is dispatched to the worker (started). + Update registry is cleared, speculative WT disappears from server. + Update is retired and second speculative WT is scheduled but not dispatched yet. + An activity completes, it converts the 2nd speculative WT into normal one. + The first speculative WT responds back, server fails request because WorkflowTaskStarted event Id is mismatched. + The second speculative WT responds back and server completes it. + */ + + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Schedule activity. + return []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: s.Tv().ActivityID(), + ActivityType: s.Tv().ActivityType(), + TaskQueue: s.Tv().TaskQueue(), + ScheduleToCloseTimeout: s.Tv().Any().InfiniteTimeout(), + }}, + }}, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + atHandler := func(task *workflowservice.PollActivityTaskQueueResponse) (*commonpb.Payloads, bool, error) { + return s.Tv().Any().Payloads(), false, nil + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + ActivityTaskHandler: atHandler, + Logger: s.Logger, + T: s.T(), + } + + // First WT will schedule activity. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + + // Send 1st update. It will create 2nd WT as speculative. + sendUpdateNoError(s, s.Tv()) + + // Poll 2nd speculative WT with 1st update. + wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt2) + s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") + s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") + s.EqualValues(7, wt2.StartedEventId) + s.EqualValues(6, wt2.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4054,28 +4089,28 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 6 WorkflowTaskScheduled 7 WorkflowTaskStarted`, wt2.History) - // Clear update registry. Speculative WFT disappears from server. - s.clearUpdateRegistryAndAbortPendingUpdates(tv) + // Clear update registry. Speculative WFT disappears from server. + clearUpdateRegistryAndAbortPendingUpdates(s, s.Tv()) - // Wait for update request to be retry by frontend and recreated in registry. This will create a 3rd WFT as speculative. - waitUpdateAdmitted(s, tv) + // Wait for update request to be retry by frontend and recreated in registry. This will create a 3rd WFT as speculative. + waitUpdateAdmitted(s, s.Tv()) - // Before polling for the 3rd speculative WT, process activity. This will convert 3rd speculative WT to normal WT. - err = poller.PollAndProcessActivityTask(false) - s.NoError(err) + // Before polling for the 3rd speculative WT, process activity. This will convert 3rd speculative WT to normal WT. + err = poller.PollAndProcessActivityTask(false) + s.NoError(err) - // Poll the 3rd WFT (not speculative anymore) but must have 2nd update. - wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - }) - s.NoError(err) - s.NotNil(wt3) - s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") - s.Len(wt3.Messages, 1, "3rd workflow task must have a message with 2nd update") - s.EqualValues(9, wt3.StartedEventId) - s.EqualValues(8, wt3.Messages[0].GetEventId()) - s.EqualHistory(` + // Poll the 3rd WFT (not speculative anymore) but must have 2nd update. + wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt3) + s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") + s.Len(wt3.Messages, 1, "3rd workflow task must have a message with 2nd update") + s.EqualValues(9, wt3.StartedEventId) + s.EqualValues(8, wt3.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4086,28 +4121,28 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 8 ActivityTaskCompleted 9 WorkflowTaskStarted`, wt3.History) - // Now try to complete 2nd WT (speculative). It should fail because WorkflowTaskStarted event Id is mismatched. - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt2.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, wt2.Messages[0]), - }) - s.Error(err, "Must fail because WorkflowTaskStarted event Id is different.") - s.IsType(&serviceerror.NotFound{}, err) - s.Contains(err.Error(), "Workflow task not found") + // Now try to complete 2nd WT (speculative). It should fail because WorkflowTaskStarted event Id is mismatched. + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt2.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), wt2.Messages[0]), + }) + s.Error(err, "Must fail because WorkflowTaskStarted event Id is different.") + s.ErrorAs(err, new(*serviceerror.NotFound)) + s.Contains(err.Error(), "Workflow task not found") - // Complete 3rd WT. It should succeed. - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt3.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, wt3.Messages[0]), - }) - s.NoError(err) + // Complete 3rd WT. It should succeed. + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt3.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), wt3.Messages[0]), + }) + s.NoError(err) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4121,61 +4156,62 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 11 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId":8} 12 WorkflowExecutionUpdateCompleted `, events) -} - -func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDifferentStartTime() { - /* - Test scenario: - An update created a speculative WT and WT is dispatched to the worker (started). - WF context is cleared, speculative WT is disappeared from server. - Update is retried and second speculative WT is dispatched to worker with same WT scheduled/started Id and update Id. - The first speculative WT respond back, server reject it because startTime is different. - The second speculative WT respond back, server accept it. - */ - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - return nil, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - Logger: s.Logger, - T: s.T(), - } - - // First WT will schedule activity. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.NotNil(res) - - // Send update. It will create 2nd WT as speculative. - sendUpdateNoError(s, tv) - - // Poll 2nd speculative WT with 1st update. - wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), }) - s.NoError(err) - s.NotNil(wt2) - s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") - s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") - s.EqualValues(6, wt2.StartedEventId) - s.EqualValues(5, wt2.Messages[0].GetEventId()) - s.EqualHistory(` + + t.Run("StaleSpeculativeWorkflowTask_Fail_BecauseOfDifferentStartTime", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + /* + Test scenario: + An update created a speculative WT and WT is dispatched to the worker (started). + WF context is cleared, speculative WT is disappeared from server. + Update is retried and second speculative WT is dispatched to worker with same WT scheduled/started Id and update Id. + The first speculative WT respond back, server reject it because startTime is different. + The second speculative WT respond back, server accept it. + */ + mustStartWorkflow(s, s.Tv()) + + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + return nil, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } + } + + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + Logger: s.Logger, + T: s.T(), + } + + // First WT will schedule activity. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.NotNil(res) + + // Send update. It will create 2nd WT as speculative. + sendUpdateNoError(s, s.Tv()) + + // Poll 2nd speculative WT with 1st update. + wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt2) + s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") + s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") + s.EqualValues(6, wt2.StartedEventId) + s.EqualValues(5, wt2.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4183,24 +4219,24 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, wt2.History) - // Clear update registry. Speculative WFT disappears from server. - s.clearUpdateRegistryAndAbortPendingUpdates(tv) + // Clear update registry. Speculative WFT disappears from server. + clearUpdateRegistryAndAbortPendingUpdates(s, s.Tv()) - // Wait for update request to be retry by frontend and recreated in registry. This will create a 3rd WFT as speculative. - waitUpdateAdmitted(s, tv) + // Wait for update request to be retry by frontend and recreated in registry. This will create a 3rd WFT as speculative. + waitUpdateAdmitted(s, s.Tv()) - // Poll for the 3rd speculative WT. - wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - }) - s.NoError(err) - s.NotNil(wt3) - s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") - s.Len(wt3.Messages, 1, "3rd workflow task must have a message with 1st update") - s.EqualValues(6, wt3.StartedEventId) - s.EqualValues(5, wt3.Messages[0].GetEventId()) - s.EqualHistory(` + // Poll for the 3rd speculative WT. + wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testcore.NewContext(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt3) + s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") + s.Len(wt3.Messages, 1, "3rd workflow task must have a message with 1st update") + s.EqualValues(6, wt3.StartedEventId) + s.EqualValues(5, wt3.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4208,28 +4244,28 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, wt3.History) - // Now try to complete 2nd (speculative) WT, it should fail. - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt2.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, wt2.Messages[0]), - }) - s.Error(err, "Must fail because workflow task start time is different.") - s.IsType(&serviceerror.NotFound{}, err) - s.Contains(err.Error(), "Workflow task not found") + // Now try to complete 2nd (speculative) WT, it should fail. + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt2.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), wt2.Messages[0]), + }) + s.Error(err, "Must fail because workflow task start time is different.") + s.ErrorAs(err, new(*serviceerror.NotFound)) + s.Contains(err.Error(), "Workflow task not found") - // Try to complete 3rd WT, it should succeed - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt3.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, wt3.Messages[0]), - }) - s.NoError(err, "2nd speculative WT should be completed because it has same WT scheduled/started Id and startTime matches the accepted message is valid (same update Id)") + // Try to complete 3rd WT, it should succeed + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testcore.NewContext(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt3.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), wt3.Messages[0]), + }) + s.NoError(err, "2nd speculative WT should be completed because it has same WT scheduled/started Id and startTime matches the accepted message is valid (same update Id)") - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4240,54 +4276,54 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_BecauseOfDif 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId":5} 9 WorkflowExecutionUpdateCompleted `, events) -} - -func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_NewWorkflowTaskWith2Updates() { - /* - Test scenario: - An update created a speculative WT and WT is dispatched to the worker (started). - Mutable state cleared, speculative WT and update registry are disappeared from server. - First update is retried and another update come in. - Second speculative WT is dispatched to worker with same WT scheduled/started Id but 2 updates. - The first speculative WT responds back, server rejected it (different start time). - The second speculative WT responds back, server accepted it. - */ - - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2) - - testCtx := testcore.NewContext() - - // Drain first WFT. - wt1, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), }) - s.NoError(err) - s.NotNil(wt1) - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt1.TaskToken, - }) - s.NoError(err) - // Send 1st update. It will create 2nd speculative WFT. - sendUpdateNoError(s, tv1) + t.Run("StaleSpeculativeWorkflowTask_Fail_NewWorkflowTaskWith2Updates", func(t *testing.T) { + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + /* + Test scenario: + An update created a speculative WT and WT is dispatched to the worker (started). + Mutable state cleared, speculative WT and update registry are disappeared from server. + First update is retried and another update come in. + Second speculative WT is dispatched to worker with same WT scheduled/started Id but 2 updates. + The first speculative WT responds back, server rejected it (different start time). + The second speculative WT responds back, server accepted it. + */ - // Poll 2nd speculative WFT with 1st update. - wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - }) - s.NoError(err) - s.NotNil(wt2) - s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") - s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") - s.EqualValues(6, wt2.StartedEventId) - s.EqualValues(5, wt2.Messages[0].GetEventId()) - s.EqualHistory(` + mustStartWorkflow(s, s.Tv()) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2) + + testCtx := testcore.NewContext() + + // Drain first WFT. + wt1, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt1) + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt1.TaskToken, + }) + s.NoError(err) + + // Send 1st update. It will create 2nd speculative WFT. + sendUpdateNoError(s, tv1) + + // Poll 2nd speculative WFT with 1st update. + wt2, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt2) + s.NotEmpty(wt2.TaskToken, "2nd workflow task must have valid task token") + s.Len(wt2.Messages, 1, "2nd workflow task must have a message with 1st update") + s.EqualValues(6, wt2.StartedEventId) + s.EqualValues(5, wt2.Messages[0].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4295,29 +4331,29 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_NewWorkflowT 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, wt2.History) - // Clear update registry. Speculative WFT disappears from server. - s.clearUpdateRegistryAndAbortPendingUpdates(tv) + // Clear update registry. Speculative WFT disappears from server. + clearUpdateRegistryAndAbortPendingUpdates(s, s.Tv()) - // Make sure UpdateWorkflowExecution call for the update "1" is retried and new (3rd) WFT is created as speculative with updateID=1. - waitUpdateAdmitted(s, tv1) + // Make sure UpdateWorkflowExecution call for the update "1" is retried and new (3rd) WFT is created as speculative with updateID=1. + waitUpdateAdmitted(s, tv1) - // Send 2nd update (with DIFFERENT updateId). It reuses already created 3rd WFT. - sendUpdateNoError(s, tv2) - // updateID=1 is still blocked. There must be 2 blocked updates now. + // Send 2nd update (with DIFFERENT updateId). It reuses already created 3rd WFT. + sendUpdateNoError(s, tv2) + // updateID=1 is still blocked. There must be 2 blocked updates now. - // Poll the 3rd speculative WFT. - wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - }) - s.NoError(err) - s.NotNil(wt3) - s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") - s.Len(wt3.Messages, 2, "3rd workflow task must have a message with 1st and 2nd updates") - s.EqualValues(6, wt3.StartedEventId) - s.EqualValues(5, wt3.Messages[0].GetEventId()) - s.EqualValues(5, wt3.Messages[1].GetEventId()) - s.EqualHistory(` + // Poll the 3rd speculative WFT. + wt3, err := s.FrontendClient().PollWorkflowTaskQueue(testCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + }) + s.NoError(err) + s.NotNil(wt3) + s.NotEmpty(wt3.TaskToken, "3rd workflow task must have valid task token") + s.Len(wt3.Messages, 2, "3rd workflow task must have a message with 1st and 2nd updates") + s.EqualValues(6, wt3.StartedEventId) + s.EqualValues(5, wt3.Messages[0].GetEventId()) + s.EqualValues(5, wt3.Messages[1].GetEventId()) + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4325,35 +4361,35 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_NewWorkflowT 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, wt3.History) - // Now try to complete 2nd speculative WT, it should fail because start time does not match. - _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt2.TaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv1), - Messages: s.UpdateAcceptCompleteMessages(tv1, wt2.Messages[0]), - ReturnNewWorkflowTask: true, - }) - s.Error(err, "Must fail because start time is different.") - s.Contains(err.Error(), "Workflow task not found") - s.IsType(&serviceerror.NotFound{}, err) + // Now try to complete 2nd speculative WT, it should fail because start time does not match. + _, err = s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt2.TaskToken, + Commands: s.UpdateAcceptCompleteCommands(tv1), + Messages: s.UpdateAcceptCompleteMessages(tv1, wt2.Messages[0]), + ReturnNewWorkflowTask: true, + }) + s.Error(err, "Must fail because start time is different.") + s.Contains(err.Error(), "Workflow task not found") + s.ErrorAs(err, new(*serviceerror.NotFound)) - // Complete of the 3rd WT should succeed. It must accept both updates. - wt4Resp, err := s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ - Namespace: s.Namespace().String(), - TaskToken: wt3.TaskToken, - Commands: append( - s.UpdateAcceptCompleteCommands(tv1), - s.UpdateAcceptCompleteCommands(tv2)...), - Messages: append( - s.UpdateAcceptCompleteMessages(tv1, wt3.Messages[0]), - s.UpdateAcceptCompleteMessages(tv2, wt3.Messages[1])...), - ReturnNewWorkflowTask: true, - }) - s.NoError(err) - s.NotNil(wt4Resp) + // Complete of the 3rd WT should succeed. It must accept both updates. + wt4Resp, err := s.FrontendClient().RespondWorkflowTaskCompleted(testCtx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: s.Namespace().String(), + TaskToken: wt3.TaskToken, + Commands: append( + s.UpdateAcceptCompleteCommands(tv1), + s.UpdateAcceptCompleteCommands(tv2)...), + Messages: append( + s.UpdateAcceptCompleteMessages(tv1, wt3.Messages[0]), + s.UpdateAcceptCompleteMessages(tv2, wt3.Messages[1])...), + ReturnNewWorkflowTask: true, + }) + s.NoError(err) + s.NotNil(wt4Resp) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4366,26 +4402,25 @@ func (s *UpdateWorkflowSuite) TestStaleSpeculativeWorkflowTask_Fail_NewWorkflowT 10 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} 11 WorkflowExecutionUpdateCompleted `, events) -} + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_WorkerSkippedProcessing_RejectByServer() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_WorkerSkippedProcessing_RejectByServer", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2) - mustStartWorkflow(s, tv) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2) + var update2ResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - var update2ResultCh <-chan *workflowservice.UpdateWorkflowExecutionResponse - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4393,91 +4428,92 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_WorkerSkippedProcessin 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted `, task.History) - update2ResultCh = sendUpdateNoError(s, tv2) - return nil, nil - case 3: - s.EqualHistory(` + update2ResultCh = sendUpdateNoError(s, tv2) + return nil, nil + case 3: + s.EqualHistory(` 4 WorkflowTaskCompleted // Speculative WT was dropped and history starts from 4 again. 5 WorkflowTaskScheduled 6 WorkflowTaskStarted`, task.History) - commands := append(s.UpdateAcceptCompleteCommands(tv2), - &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, - }) - return commands, nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + commands := append(s.UpdateAcceptCompleteCommands(tv2), + &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}}, + }) + return commands, nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+tv1.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.EqualValues(5, updRequestMsg.GetEventId()) - // Don't process update in WT. - return nil, nil - case 3: - s.Len(task.Messages, 1) - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + // Don't process update in WT. + return nil, nil + case 3: + s.Len(task.Messages, 1) + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv2, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + s.Equal("args-value-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.EqualValues(5, updRequestMsg.GetEventId()) + return s.UpdateAcceptCompleteMessages(tv2, updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Identity: "old_worker", - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Identity: "old_worker", + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv1) + updateResultCh := sendUpdateNoError(s, tv1) - // Process 2nd WT which ignores update message. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.Equal("Workflow Update is rejected because it wasn't processed by worker. Probably, Workflow Update is not supported by the worker.", updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualValues(3, updateResp.ResetHistoryEventId) + // Process 2nd WT which ignores update message. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("Workflow Update is rejected because it wasn't processed by worker. Probably, Workflow Update is not supported by the worker.", updateResult.GetOutcome().GetFailure().GetMessage()) + s.EqualValues(3, updateResp.ResetHistoryEventId) - // Process 3rd WT which completes 2nd update and workflow. - update2Resp, err := poller.HandlePartialWorkflowTask(updateResp.GetWorkflowTask(), false) - s.NoError(err) - s.NotNil(update2Resp) - update2Result := <-update2ResultCh - s.EqualValues("success-result-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), update2Result.GetOutcome().GetSuccess())) + // Process 3rd WT which completes 2nd update and workflow. + update2Resp, err := poller.HandlePartialWorkflowTask(updateResp.GetWorkflowTask(), false) + s.NoError(err) + s.NotNil(update2Resp) + update2Result := <-update2ResultCh + s.Equal("success-result-of-"+tv2.UpdateID(), testcore.DecodeString(s.T(), update2Result.GetOutcome().GetSuccess())) - s.Equal(3, wtHandlerCalls) - s.Equal(3, msgHandlerCalls) + s.Equal(3, wtHandlerCalls) + s.Equal(3, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(` + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4488,66 +4524,66 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_WorkerSkippedProcessin 8 WorkflowExecutionUpdateAccepted 9 WorkflowExecutionUpdateCompleted 10 WorkflowExecutionCompleted`, events) -} + }) -func (s *UpdateWorkflowSuite) TestLastWorkflowTask_HasUpdateMessage() { - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) + t.Run("LastWorkflowTask_HasUpdateMessage", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - completeWorkflowCommand := &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ - Result: tv.Any().Payloads(), + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + completeWorkflowCommand := &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ + Result: s.Tv().Any().Payloads(), + }, }, - }, - } - return append(s.UpdateAcceptCommands(tv), completeWorkflowCommand), nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - return s.UpdateAcceptMessages(tv, task.Messages[0]), nil - }, - Logger: s.Logger, - T: s.T(), - } + } + return append(s.UpdateAcceptCommands(s.Tv()), completeWorkflowCommand), nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + return s.UpdateAcceptMessages(s.Tv(), task.Messages[0]), nil + }, + Logger: s.Logger, + T: s.T(), + } - updateResultCh := sendUpdateNoErrorWaitPolicyAccepted(s, tv) - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResult := <-updateResultCh - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult.GetStage()) - s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult.GetOutcome().GetFailure().GetMessage()) + updateResultCh := sendUpdateNoErrorWaitPolicyAccepted(s, s.Tv()) + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResult := <-updateResultCh + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult.GetStage()) + s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", updateResult.GetOutcome().GetFailure().GetMessage()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowExecutionUpdateAccepted 6 WorkflowExecutionCompleted - `, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) -} + `, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) -func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_QueryFailureClearsWFContext() { - tv := testvars.New(s.T()) + t.Run("SpeculativeWorkflowTask_QueryFailureClearsWFContext", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - mustStartWorkflow(s, tv) - - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - s.EqualHistory(` + wtHandlerCalls := 0 + wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + wtHandlerCalls++ + switch wtHandlerCalls { + case 1: + // Completes first WT with empty command list. + return nil, nil + case 2: + s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted @@ -4555,343 +4591,347 @@ func (s *UpdateWorkflowSuite) TestSpeculativeWorkflowTask_QueryFailureClearsWFCo 5 WorkflowTaskScheduled 6 WorkflowTaskStarted `, task.History) - return s.UpdateAcceptCompleteCommands(tv), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + default: + s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) + return nil, nil + } } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: - updRequestMsg := task.Messages[0] - updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) + msgHandlerCalls := 0 + msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + msgHandlerCalls++ + switch msgHandlerCalls { + case 1: + return nil, nil + case 2: + updRequestMsg := task.Messages[0] + updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) - s.Equal("args-value-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) - s.Equal(tv.HandlerName(), updRequest.GetInput().GetName()) - s.EqualValues(5, updRequestMsg.GetEventId()) + s.Equal("args-value-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updRequest.GetInput().GetArgs())) + s.Equal(s.Tv().HandlerName(), updRequest.GetInput().GetName()) + s.EqualValues(5, updRequestMsg.GetEventId()) - return s.UpdateAcceptCompleteMessages(tv, updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil + return s.UpdateAcceptCompleteMessages(s.Tv(), updRequestMsg), nil + default: + s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) + return nil, nil + } } - } - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + WorkflowTaskHandler: wtHandler, + MessageHandler: msgHandler, + Logger: s.Logger, + T: s.T(), + } - // Drain first WT. - _, err := poller.PollAndProcessWorkflowTask() - s.NoError(err) + // Drain first WT. + _, err := poller.PollAndProcessWorkflowTask() + s.NoError(err) - updateResultCh := sendUpdateNoError(s, tv) + updateResultCh := sendUpdateNoError(s, s.Tv()) - type QueryResult struct { - Resp *workflowservice.QueryWorkflowResponse - Err error - } - queryFn := func(resCh chan<- QueryResult) { - // There is no query handler, and query timeout is ok for this test. - // But first query must not time out before 2nd query reached server, - // because 2 queries overflow the query buffer (default size 1), - // which leads to clearing of WF context. - shortCtx, cancel := context.WithTimeout(testcore.NewContext(), 100*time.Millisecond) - defer cancel() - queryResp, err := s.FrontendClient().QueryWorkflow(shortCtx, &workflowservice.QueryWorkflowRequest{ - Namespace: s.Namespace().String(), - Execution: tv.WorkflowExecution(), - Query: &querypb.WorkflowQuery{ - QueryType: tv.Any().String(), - }, - }) - resCh <- QueryResult{Resp: queryResp, Err: err} - } + type QueryResult struct { + Resp *workflowservice.QueryWorkflowResponse + Err error + } + queryFn := func(resCh chan<- QueryResult) { + // There is no query handler, and query timeout is ok for this test. + // But first query must not time out before 2nd query reached server, + // because 2 queries overflow the query buffer (default size 1), + // which leads to clearing of WF context. + shortCtx, cancel := context.WithTimeout(testcore.NewContext(), 100*time.Millisecond) + defer cancel() + queryResp, err := s.FrontendClient().QueryWorkflow(shortCtx, &workflowservice.QueryWorkflowRequest{ + Namespace: s.Namespace().String(), + Execution: s.Tv().WorkflowExecution(), + Query: &querypb.WorkflowQuery{ + QueryType: s.Tv().Any().String(), + }, + }) + resCh <- QueryResult{Resp: queryResp, Err: err} + } - query1ResultCh := make(chan QueryResult) - query2ResultCh := make(chan QueryResult) - go queryFn(query1ResultCh) - go queryFn(query2ResultCh) - query1Res := <-query1ResultCh - query2Res := <-query2ResultCh - s.Error(query1Res.Err) - s.Error(query2Res.Err) - s.Nil(query1Res.Resp) - s.Nil(query2Res.Resp) + query1ResultCh := make(chan QueryResult) + query2ResultCh := make(chan QueryResult) + go queryFn(query1ResultCh) + go queryFn(query2ResultCh) + query1Res := <-query1ResultCh + query2Res := <-query2ResultCh + s.Error(query1Res.Err) + s.Error(query2Res.Err) + s.Nil(query1Res.Resp) + s.Nil(query2Res.Resp) - var queryBufferFullErr *serviceerror.ResourceExhausted - if common.IsContextDeadlineExceededErr(query1Res.Err) { - s.True(common.IsContextDeadlineExceededErr(query1Res.Err), "one of query errors must be CDE") - s.ErrorAs(query2Res.Err, &queryBufferFullErr, "one of query errors must `query buffer is full`") - s.Contains(query2Res.Err.Error(), "query buffer is full", "one of query errors must `query buffer is full`") - } else { - s.ErrorAs(query1Res.Err, &queryBufferFullErr, "one of query errors must `query buffer is full`") - s.Contains(query1Res.Err.Error(), "query buffer is full", "one of query errors must `query buffer is full`") - s.True(common.IsContextDeadlineExceededErr(query2Res.Err), "one of query errors must be CDE") - } + var queryBufferFullErr *serviceerror.ResourceExhausted + if common.IsContextDeadlineExceededErr(query1Res.Err) { + s.True(common.IsContextDeadlineExceededErr(query1Res.Err), "one of query errors must be CDE") + s.ErrorAs(query2Res.Err, &queryBufferFullErr, "one of query errors must `query buffer is full`") + s.Contains(query2Res.Err.Error(), "query buffer is full", "one of query errors must `query buffer is full`") + } else { + s.ErrorAs(query1Res.Err, &queryBufferFullErr, "one of query errors must `query buffer is full`") + s.Contains(query1Res.Err.Error(), "query buffer is full", "one of query errors must `query buffer is full`") + s.True(common.IsContextDeadlineExceededErr(query2Res.Err), "one of query errors must be CDE") + } - // "query buffer is full" error clears WF context. If update registry is not cleared together with context (old behaviour), - // then update stays there but speculative WFT which supposed to deliver it, is cleared. - // Subsequent retry attempts of "UpdateWorkflowExecution" API wouldn't help, because update is deduped by registry, - // and new WFT is not created. Update is not delivered to the worker until new WFT is created. - // If registry is cleared together with WF context (current behaviour), retries of "UpdateWorkflowExecution" - // will create new update and WFT. + // "query buffer is full" error clears WF context. If update registry is not cleared together with context (old behaviour), + // then update stays there but speculative WFT which supposed to deliver it, is cleared. + // Subsequent retry attempts of "UpdateWorkflowExecution" API wouldn't help, because update is deduped by registry, + // and new WFT is not created. Update is not delivered to the worker until new WFT is created. + // If registry is cleared together with WF context (current behaviour), retries of "UpdateWorkflowExecution" + // will create new update and WFT. - // Wait to make sure that UpdateWorkflowExecution call is retried, update and speculative WFT are recreated. - time.Sleep(500 * time.Millisecond) //nolint:forbidigo + // Wait to make sure that UpdateWorkflowExecution call is retried, update and speculative WFT are recreated. + time.Sleep(500 * time.Millisecond) //nolint:forbidigo - // Process update in workflow. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResp := res.NewTask - updateResult := <-updateResultCh - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, updateResp.ResetHistoryEventId) + // Process update in workflow. + res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResp := res.NewTask + updateResult := <-updateResultCh + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + s.EqualValues(0, updateResp.ResetHistoryEventId) - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) + s.Equal(2, wtHandlerCalls) + s.Equal(2, msgHandlerCalls) - events := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) + events := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowTaskScheduled // Was speculative WT... 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted // ...and events were written to the history when WT completes. + 7 WorkflowTaskCompleted // ...and events were written to the history when WT completes. 8 WorkflowExecutionUpdateAccepted 9 WorkflowExecutionUpdateCompleted `, events) -} + }) -func (s *UpdateWorkflowSuite) TestUpdatesAreSentToWorkerInOrderOfAdmission() { - // If our implementation is not in fact ordering updates correctly, then it may be ordering them - // non-deterministically. This number should be high enough that the false-negative rate of the test is low, but - // must not exceed our limit on number of in-flight updates. If we were picking a random ordering then the - // false-negative rate would be 1/(nUpdates!). - nUpdates := 10 + t.Run("UpdatesAreSentToWorkerInOrderOfAdmission", func(t *testing.T) { + s := testcore.NewEnv(t) + // If our implementation is not in fact ordering updates correctly, then it may be ordering them + // non-deterministically. This number should be high enough that the false-negative rate of the test is low, but + // must not exceed our limit on number of in-flight updates. If we were picking a random ordering then the + // false-negative rate would be 1/(nUpdates!). + nUpdates := 10 - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - for i := 0; i < nUpdates; i++ { - // Sequentially send updates one by one. - sendUpdateNoError(s, tv.WithUpdateIDNumber(i)) - } + mustStartWorkflow(s, s.Tv()) + for i := 0; i < nUpdates; i++ { + // Sequentially send updates one by one. + sendUpdateNoError(s, s.Tv().WithUpdateIDNumber(i)) + } - wtHandlerCalls := 0 - msgHandlerCalls := 0 - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - msgHandlerCalls++ - var commands []*commandpb.Command - for i := range task.Messages { - commands = append(commands, s.UpdateAcceptCompleteCommands(tv.WithMessageIDNumber(i))...) - } - return commands, nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - wtHandlerCalls++ - s.Len(task.Messages, nUpdates) - var messages []*protocolpb.Message - // Updates were sent in sequential order of updateId => messages must be ordered in the same way. - for i, m := range task.Messages { - s.Equal(tv.WithUpdateIDNumber(i).UpdateID(), m.ProtocolInstanceId) - messages = append(messages, s.UpdateAcceptCompleteMessages(tv.WithMessageIDNumber(i), m)...) - } - return messages, nil - }, - Logger: s.Logger, - T: s.T(), - } - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - s.Equal(1, wtHandlerCalls) - s.Equal(1, msgHandlerCalls) + wtHandlerCalls := 0 + msgHandlerCalls := 0 + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + msgHandlerCalls++ + var commands []*commandpb.Command + for i := range task.Messages { + commands = append(commands, s.UpdateAcceptCompleteCommands(s.Tv().WithMessageIDNumber(i))...) + } + return commands, nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + wtHandlerCalls++ + s.Len(task.Messages, nUpdates) + var messages []*protocolpb.Message + // Updates were sent in sequential order of updateId => messages must be ordered in the same way. + for i, m := range task.Messages { + s.Equal(s.Tv().WithUpdateIDNumber(i).UpdateID(), m.ProtocolInstanceId) + messages = append(messages, s.UpdateAcceptCompleteMessages(s.Tv().WithMessageIDNumber(i), m)...) + } + return messages, nil + }, + Logger: s.Logger, + T: s.T(), + } + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + s.Equal(1, wtHandlerCalls) + s.Equal(1, msgHandlerCalls) - expectedHistory := ` + expectedHistory := ` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted ` - for i := 0; i < nUpdates; i++ { - tvi := tv.WithUpdateIDNumber(i) - expectedHistory += fmt.Sprintf(` + for i := 0; i < nUpdates; i++ { + tvi := s.Tv().WithUpdateIDNumber(i) + expectedHistory += fmt.Sprintf(` %d WorkflowExecutionUpdateAccepted {"AcceptedRequest":{"Meta": {"UpdateId": "%s"}}} %d WorkflowExecutionUpdateCompleted {"Meta": {"UpdateId": "%s"}}`, - 5+2*i, tvi.UpdateID(), - 6+2*i, tvi.UpdateID()) - } + 5+2*i, tvi.UpdateID(), + 6+2*i, tvi.UpdateID()) + } - history := s.GetHistory(s.Namespace().String(), tv.WorkflowExecution()) - s.EqualHistoryEvents(expectedHistory, history) -} + history := s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution()) + s.EqualHistoryEvents(expectedHistory, history) + }) -func (s *UpdateWorkflowSuite) TestWaitAccepted_GotCompleted() { - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) + t.Run("WaitAccepted_GotCompleted", func(t *testing.T) { + s := testcore.NewEnv(t) + mustStartWorkflow(s, s.Tv()) - poller := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - return s.UpdateAcceptCompleteCommands(tv), nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - return s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), nil - }, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + return s.UpdateAcceptCompleteCommands(s.Tv()), nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + return s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), nil + }, + Logger: s.Logger, + T: s.T(), + } - // Send Update with intent to wait for Accepted stage only, - updateResultCh := sendUpdateNoErrorWaitPolicyAccepted(s, tv) - _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) - s.NoError(err) - updateResult := <-updateResultCh - // but Update was accepted and completed on the same WFT, and outcome was returned. - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult.GetStage()) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) + // Send Update with intent to wait for Accepted stage only, + updateResultCh := sendUpdateNoErrorWaitPolicyAccepted(s, s.Tv()) + _, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + s.NoError(err) + updateResult := <-updateResultCh + // but Update was accepted and completed on the same WFT, and outcome was returned. + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, updateResult.GetStage()) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowExecutionUpdateAccepted 6 WorkflowExecutionUpdateCompleted - `, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) -} + `, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) -func (s *UpdateWorkflowSuite) TestContinueAsNew_UpdateIsNotCarriedOver() { - tv := testvars.New(s.T()) - firstRunID := mustStartWorkflow(s, tv) - tv1 := tv.WithUpdateIDNumber(1).WithMessageIDNumber(1) - tv2 := tv.WithUpdateIDNumber(2).WithMessageIDNumber(2) + t.Run("ContinueAsNew_UpdateIsNotCarriedOver", func(t *testing.T) { + s := testcore.NewEnv(t) + firstRunID := mustStartWorkflow(s, s.Tv()) + tv1 := s.Tv().WithUpdateIDNumber(1).WithMessageIDNumber(1) + tv2 := s.Tv().WithUpdateIDNumber(2).WithMessageIDNumber(2) - /* - 1st Update goes to the 1st run and accepted (but not completed) by Workflow. - While this WFT is running, 2nd Update is sent, and WFT is completing with CAN for the 1st run. - There are 2 Updates in the registry of the 1st run: 1st is accepted and 2nd is admitted. - Both of them are aborted but with different errors: - - Admitted Update is aborted with retryable "workflow is closing" error. SDK should retry this error - and new attempt should land on the new run. - - Accepted Update is aborted with update failure. - */ + /* + 1st Update goes to the 1st run and accepted (but not completed) by Workflow. + While this WFT is running, 2nd Update is sent, and WFT is completing with CAN for the 1st run. + There are 2 Updates in the registry of the 1st run: 1st is accepted and 2nd is admitted. + Both of them are aborted but with different errors: + - Admitted Update is aborted with retryable "workflow is closing" error. SDK should retry this error + and new attempt should land on the new run. + - Accepted Update is aborted with update failure. + */ - var update2ResponseCh <-chan updateResponseErr + var update2ResponseCh <-chan updateResponseErr - poller1 := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - // Send 2nd Update while WFT is running. - update2ResponseCh = sendUpdate(context.Background(), s, tv2) - canCommand := &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.WithTaskQueueNumber(2).TaskQueue(), - }}, - } - return append(s.UpdateAcceptCommands(tv1), canCommand), nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - return s.UpdateAcceptMessages(tv1, task.Messages[0]), nil - }, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller1 := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + // Send 2nd Update while WFT is running. + update2ResponseCh = sendUpdate(context.Background(), s, tv2) + canCommand := &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: s.Tv().WorkflowType(), + TaskQueue: s.Tv().WithTaskQueueNumber(2).TaskQueue(), + }}, + } + return append(s.UpdateAcceptCommands(tv1), canCommand), nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + return s.UpdateAcceptMessages(tv1, task.Messages[0]), nil + }, + Logger: s.Logger, + T: s.T(), + } - poller2 := &testcore.TaskPoller{ - Client: s.FrontendClient(), - Namespace: s.Namespace().String(), - TaskQueue: tv.WithTaskQueueNumber(2).TaskQueue(), - Identity: tv.WorkerIdentity(), - WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - return nil, nil - }, - MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - s.Empty(task.Messages, "no Updates should be carried over to the 2nd run") - return nil, nil - }, - Logger: s.Logger, - T: s.T(), - } + //nolint:staticcheck // SA1019 TaskPoller replacement needed + poller2 := &testcore.TaskPoller{ + Client: s.FrontendClient(), + Namespace: s.Namespace().String(), + TaskQueue: s.Tv().WithTaskQueueNumber(2).TaskQueue(), + Identity: s.Tv().WorkerIdentity(), + WorkflowTaskHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { + return nil, nil + }, + MessageHandler: func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { + s.Empty(task.Messages, "no Updates should be carried over to the 2nd run") + return nil, nil + }, + Logger: s.Logger, + T: s.T(), + } - update1ResponseCh := sendUpdate(context.Background(), s, tv1) - _, err := poller1.PollAndProcessWorkflowTask() - s.NoError(err) + update1ResponseCh := sendUpdate(context.Background(), s, tv1) + _, err := poller1.PollAndProcessWorkflowTask() + s.NoError(err) - _, err = poller2.PollAndProcessWorkflowTask() - s.NoError(err) + _, err = poller2.PollAndProcessWorkflowTask() + s.NoError(err) - update1Response := <-update1ResponseCh - s.NoError(update1Response.err) - s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", update1Response.response.GetOutcome().GetFailure().GetMessage()) + update1Response := <-update1ResponseCh + s.NoError(update1Response.err) + s.Equal("Workflow Update failed because the Workflow completed before the Update completed.", update1Response.response.GetOutcome().GetFailure().GetMessage()) - update2Response := <-update2ResponseCh - s.Error(update2Response.err) - var resourceExhausted *serviceerror.ResourceExhausted - s.ErrorAs(update2Response.err, &resourceExhausted) - s.Equal("workflow operation can not be applied because workflow is closing", update2Response.err.Error()) + update2Response := <-update2ResponseCh + s.Error(update2Response.err) + var resourceExhausted *serviceerror.ResourceExhausted + s.ErrorAs(update2Response.err, &resourceExhausted) + s.Equal("workflow operation can not be applied because workflow is closing", update2Response.err.Error()) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionContinuedAsNew`, s.GetHistory(s.Namespace().String(), tv.WithRunID(firstRunID).WorkflowExecution())) + 6 WorkflowExecutionContinuedAsNew`, s.GetHistory(s.Namespace().String(), s.Tv().WithRunID(firstRunID).WorkflowExecution())) - s.EqualHistoryEvents(` + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted`, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) -} + 4 WorkflowTaskCompleted`, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) -func (s *UpdateWorkflowSuite) TestContinueAsNew_Suggestion() { - // setup CAN suggestion to be at 2nd Update - cleanup1 := s.OverrideDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdates, 3) - defer cleanup1() - cleanup2 := s.OverrideDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdatesSuggestContinueAsNewThreshold, 0.5) - defer cleanup2() + t.Run("ContinueAsNew_Suggestion", func(t *testing.T) { + // setup CAN suggestion to be at 2nd Update + s := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdates, 3), + testcore.WithDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdatesSuggestContinueAsNewThreshold, 0.5), + ) - // start workflow - tv := testvars.New(s.T()) - mustStartWorkflow(s, tv) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + // start workflow + mustStartWorkflow(s, s.Tv()) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) - // send Update #1 - no CAN suggested - tv1 := tv.WithUpdateIDNumber(1) - updateResultCh := sendUpdateNoError(s, tv1) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv1, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistoryEvents(` + // send Update #1 - no CAN suggested + tv1 := s.Tv().WithUpdateIDNumber(1) + updateResultCh := sendUpdateNoError(s, tv1) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv1, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted {"SuggestContinueAsNew": false} @@ -4899,154 +4939,294 @@ func (s *UpdateWorkflowSuite) TestContinueAsNew_Suggestion() { 5 WorkflowTaskScheduled 6 WorkflowTaskStarted {"SuggestContinueAsNew": false}`, task.History.Events) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv1, task.Messages[0]), - }, nil - }) - s.NoError(err) - <-updateResultCh + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(tv1, task.Messages[0]), + }, nil + }) + s.NoError(err) + <-updateResultCh - // send Update #2 - CAN suggested - tv2 := tv.WithUpdateIDNumber(2) - updateResultCh = sendUpdateNoError(s, tv2) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv2, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.EqualHistoryEventsSuffix(` + // send Update #2 - CAN suggested + tv2 := s.Tv().WithUpdateIDNumber(2) + updateResultCh = sendUpdateNoError(s, tv2) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv2, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.EqualHistoryEventsSuffix(` WorkflowTaskStarted {"SuggestContinueAsNew": true}`, task.History.Events) - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv2, task.Messages[0]), - }, nil - }) - s.NoError(err) - <-updateResultCh -} + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(tv2, task.Messages[0]), + }, nil + }) + s.NoError(err) + <-updateResultCh + }) -func (s *UpdateWorkflowSuite) TestUpdateWithStart() { - type multiopsResponseErr struct { - response *workflowservice.ExecuteMultiOperationResponse - err error - } - - sendUpdateWithStart := func( - ctx context.Context, - startReq *workflowservice.StartWorkflowExecutionRequest, - updateReq *workflowservice.UpdateWorkflowExecutionRequest, - ) chan multiopsResponseErr { - capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() - defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - - retCh := make(chan multiopsResponseErr) - go func() { - resp, err := s.FrontendClient().ExecuteMultiOperation( - ctx, - &workflowservice.ExecuteMultiOperationRequest{ - Namespace: s.Namespace().String(), - Operations: []*workflowservice.ExecuteMultiOperationRequest_Operation{ - { - Operation: &workflowservice.ExecuteMultiOperationRequest_Operation_StartWorkflow{ - StartWorkflow: startReq, - }, - }, - { - Operation: &workflowservice.ExecuteMultiOperationRequest_Operation_UpdateWorkflow{ - UpdateWorkflow: updateReq, - }, - }, - }, - }) - - if err == nil { - s.Len(resp.Responses, 2) - - startRes := resp.Responses[0].Response.(*workflowservice.ExecuteMultiOperationResponse_Response_StartWorkflow).StartWorkflow - s.NotZero(startRes.RunId) - - updateRes := resp.Responses[1].Response.(*workflowservice.ExecuteMultiOperationResponse_Response_UpdateWorkflow).UpdateWorkflow - if updateReq.WaitPolicy.LifecycleStage == enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED { - s.NotNil(updateRes.Outcome) - s.NotZero(updateRes.Outcome.String()) - } - } - - // make sure there's no lock contention - s.Empty(capture.Snapshot()[metrics.TaskWorkflowBusyCounter.Name()]) - - retCh <- multiopsResponseErr{resp, err} - }() - return retCh - } - - startWorkflowReq := func(tv *testvars.TestVars) *workflowservice.StartWorkflowExecutionRequest { - return &workflowservice.StartWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowId: tv.WorkflowID(), - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), - Identity: tv.WorkerIdentity(), + t.Run("UpdateWithStart", func(t *testing.T) { + type multiopsResponseErr struct { + response *workflowservice.ExecuteMultiOperationResponse + err error } - } - s.Run("workflow is not running", func() { + sendUpdateWithStart := func(s testcore.Env, ctx context.Context, startReq *workflowservice.StartWorkflowExecutionRequest, updateReq *workflowservice.UpdateWorkflowExecutionRequest) chan multiopsResponseErr { + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - for _, p := range []enumspb.WorkflowIdConflictPolicy{ - enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - } { - s.Run(fmt.Sprintf("start workflow and send update (with conflict policy %v)", p), func() { + retCh := make(chan multiopsResponseErr) + go func() { + resp, err := s.FrontendClient().ExecuteMultiOperation( + ctx, + &workflowservice.ExecuteMultiOperationRequest{ + Namespace: s.Namespace().String(), + Operations: []*workflowservice.ExecuteMultiOperationRequest_Operation{ + { + Operation: &workflowservice.ExecuteMultiOperationRequest_Operation_StartWorkflow{ + StartWorkflow: startReq, + }, + }, + { + Operation: &workflowservice.ExecuteMultiOperationRequest_Operation_UpdateWorkflow{ + UpdateWorkflow: updateReq, + }, + }, + }, + }) - s.Run("and accept", func() { - tv := testvars.New(s.T()) + if err == nil { + // Use assert (not require) in goroutine - require calls t.FailNow() which panics + //nolint:testifylint // intentional use of assert in goroutine + assert.Len(s.T(), resp.Responses, 2) - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = p - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + startRes := resp.Responses[0].Response.(*workflowservice.ExecuteMultiOperationResponse_Response_StartWorkflow).StartWorkflow + assert.NotEmpty(s.T(), startRes.RunId) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) + updateRes := resp.Responses[1].Response.(*workflowservice.ExecuteMultiOperationResponse_Response_UpdateWorkflow).UpdateWorkflow + if updateReq.WaitPolicy.LifecycleStage == enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED { + assert.NotNil(s.T(), updateRes.Outcome) + assert.NotEmpty(s.T(), updateRes.Outcome.String()) + } + } - uwsRes := <-uwsCh - s.NoError(err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - requireStartedAndRunning(s.T(), startResp) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + // make sure there's no lock contention + //nolint:testifylint // intentional use of assert in goroutine + assert.Empty(s.T(), capture.Snapshot()[metrics.TaskWorkflowBusyCounter.Name()]) - // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) - s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) + retCh <- multiopsResponseErr{resp, err} + }() + return retCh + } - s.EqualHistoryEvents(` + startWorkflowReq := func(s testcore.Env, tv *testvars.TestVars) *workflowservice.StartWorkflowExecutionRequest { + return &workflowservice.StartWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowId: tv.WorkflowID(), + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + Identity: tv.WorkerIdentity(), + } + } + + t.Run("workflow is not running", func(t *testing.T) { + for _, p := range []enumspb.WorkflowIdConflictPolicy{ + enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + } { + t.Run(fmt.Sprintf("start workflow and send update (with conflict policy %v)", p), func(t *testing.T) { + t.Run("and accept", func(t *testing.T) { + s := testcore.NewEnv(t) + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = p + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + requireStartedAndRunning(s.T(), startResp) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + + // poll update to ensure same outcome is returned + pollRes, err := pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.NoError(err) + s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) + + s.EqualHistoryEvents(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled 3 WorkflowTaskStarted 4 WorkflowTaskCompleted 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionUpdateCompleted`, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + 6 WorkflowExecutionUpdateCompleted`, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) + + t.Run("and reject", func(t *testing.T) { + s := testcore.NewEnv(t) + startReq := startWorkflowReq(s, s.Tv()) + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + requireStartedAndRunning(s.T(), startResp) + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateRep.GetOutcome().GetFailure().GetMessage()) + + // poll update to ensure same outcome is returned + _, err = pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.Error(err) + s.ErrorAs(err, new(*serviceerror.NotFound)) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted`, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) }) + } + }) - s.Run("and reject", func() { - tv := testvars.New(s.T()) + t.Run("workflow is running", func(t *testing.T) { + t.Run("workflow id conflict policy use-existing: only send update", func(t *testing.T) { + t.Run("and accept", func(t *testing.T) { + s := testcore.NewEnv(t) + // start workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) - startReq := startWorkflowReq(tv) - updateReq := updateWorkflowRequest(s, tv, + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, task.Messages[0]), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + requireNotStartedButRunning(s.T(), startResp) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + + // poll update to ensure same outcome is returned + pollRes, err := pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.Nil(err) + s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + 5 WorkflowTaskScheduled + 6 WorkflowTaskStarted + 7 WorkflowTaskCompleted + 8 WorkflowExecutionUpdateAccepted + 9 WorkflowExecutionUpdateCompleted`, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) + + t.Run("and reject", func(t *testing.T) { + s := testcore.NewEnv(t) + // start workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateRejectMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + requireNotStartedButRunning(s.T(), startResp) + s.Equal("rejection-of-"+s.Tv().UpdateID(), updateRep.GetOutcome().GetFailure().GetMessage()) + + // poll update to ensure same outcome is returned + _, err = pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.Error(err) + s.ErrorAs(err, new(*serviceerror.NotFound)) + + s.EqualHistoryEvents(` + 1 WorkflowExecutionStarted + 2 WorkflowTaskScheduled + 3 WorkflowTaskStarted + 4 WorkflowTaskCompleted + `, s.GetHistory(s.Namespace().String(), s.Tv().WorkflowExecution())) + }) + }) + + t.Run("workflow id conflict policy terminate-existing", func(t *testing.T) { + t.Run("terminate workflow first, then start and update", func(t *testing.T) { + s := testcore.NewEnv(t) + // start workflow + firstWF, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), }, nil }) s.NoError(err) @@ -5056,148 +5236,206 @@ func (s *UpdateWorkflowSuite) TestUpdateWithStart() { startResp := uwsRes.response.Responses[0].GetStartWorkflow() updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() requireStartedAndRunning(s.T(), startResp) - s.Equal("rejection-of-"+tv.UpdateID(), updateRep.GetOutcome().GetFailure().GetMessage()) + s.Equal(startResp.RunId, updateRep.UpdateRef.GetWorkflowExecution().RunId) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + + // ensure workflow was terminated + descResp, err := s.FrontendClient().DescribeWorkflowExecution(testcore.NewContext(), + &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: startReq.WorkflowId, RunId: firstWF.RunId}, + }) + s.NoError(err) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, descResp.WorkflowExecutionInfo.Status) // poll update to ensure same outcome is returned - _, err = pollUpdate(s, tv, + pollRes, err := pollUpdate(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Error(err) - s.IsType(&serviceerror.NotFound{}, err) + s.NoError(err) + s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) + }) - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted`, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + t.Run("given an accepted update, attach to it", func(t *testing.T) { + s := testcore.NewEnv(t) + // 1st update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh1 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + // accept the update + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + + uwsRes1 := <-uwsCh1 + s.NoError(uwsRes1.err) + startResp1 := uwsRes1.response.Responses[0].GetStartWorkflow() + updateRep1 := uwsRes1.response.Responses[1].GetUpdateWorkflow() + s.True(startResp1.Started) + s.Equal(startResp1.RunId, updateRep1.UpdateRef.GetWorkflowExecution().RunId) + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED, updateRep1.Stage) + + // 2nd update-with-start: attaches to update instead of terminating workflow + uwsCh2 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + uwsRes2 := <-uwsCh2 + s.NoError(uwsRes2.err) + startResp2 := uwsRes2.response.Responses[0].GetStartWorkflow() + updateRep2 := uwsRes2.response.Responses[1].GetUpdateWorkflow() + s.False(startResp2.Started) + s.Equal(startResp2.RunId, startResp1.RunId) // no termination + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED, updateRep2.Stage) }) }) - } - }) - s.Run("workflow is running", func() { - - s.Run("workflow id conflict policy use-existing: only send update", func() { - - s.Run("and accept", func() { - tv := testvars.New(s.T()) - - // start workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) + t.Run("workflow id conflict policy fail: abort multi operation", func(t *testing.T) { + s := testcore.NewEnv(t) + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) s.NoError(err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) + // start workflow + startWorkflowReq(s, s.Tv()) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) s.NoError(err) // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL + updateReq := updateWorkflowRequest(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) uwsRes := <-uwsCh - s.NoError(uwsRes.err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - requireNotStartedButRunning(s.T(), startResp) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) - - // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) - s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) - - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - 5 WorkflowTaskScheduled - 6 WorkflowTaskStarted - 7 WorkflowTaskCompleted - 8 WorkflowExecutionUpdateAccepted - 9 WorkflowExecutionUpdateCompleted`, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + s.Error(uwsRes.err) + s.Equal("Update-with-Start could not be executed.", uwsRes.err.Error()) + errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted + s.ErrorAs(errs[0], &alreadyStartedErr) + s.Equal("Operation was aborted.", errs[1].Error()) }) - s.Run("and reject", func() { - tv := testvars.New(s.T()) + t.Run("receive completed update result", func(t *testing.T) { + _ = testcore.NewEnv(t) // unused s + for _, p := range []enumspb.WorkflowIdConflictPolicy{ + enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + } { + t.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func(t *testing.T) { + s := testcore.NewEnv(t) + startReq := startWorkflowReq(s, s.Tv()) + updReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - // start workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) + // 1st update-with-start + uwsCh1 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updReq) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + uwsRes1 := <-uwsCh1 + s.NoError(uwsRes1.err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) + // 2nd update-with-start: using *same* UpdateID - but *different* RequestID + uwsRes2 := <-sendUpdateWithStart(s, testcore.NewContext(), startReq, updReq) + s.NoError(uwsRes2.err) - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + s.Equal(uwsRes1.response.Responses[0].GetStartWorkflow().RunId, uwsRes2.response.Responses[0].GetStartWorkflow().RunId) + s.Equal(uwsRes1.response.Responses[1].GetUpdateWorkflow().Outcome.String(), uwsRes2.response.Responses[1].GetUpdateWorkflow().Outcome.String()) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateRejectMessages(tv, task.Messages[0]), - }, nil + // poll update to ensure same outcome is returned + pollRes, err := pollUpdate(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + s.NoError(err) + s.Equal(uwsRes1.response.Responses[1].GetUpdateWorkflow().Outcome.String(), pollRes.Outcome.String()) }) - s.NoError(err) + } + }) - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - requireNotStartedButRunning(s.T(), startResp) - s.Equal("rejection-of-"+tv.UpdateID(), updateRep.GetOutcome().GetFailure().GetMessage()) + t.Run("dedupes start", func(t *testing.T) { + _ = testcore.NewEnv(t) // unused s + for _, p := range []enumspb.WorkflowIdConflictPolicy{ + enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + } { + t.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func(t *testing.T) { + s := testcore.NewEnv(t) + startReq := startWorkflowReq(s, s.Tv()) + startReq.RequestId = "request_id" + startReq.WorkflowIdConflictPolicy = p + updReq1 := updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(1), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - // poll update to ensure same outcome is returned - _, err = pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Error(err) - s.IsType(&serviceerror.NotFound{}, err) + // 1st update-with-start + uwsCh1 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updReq1) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + uwsRes1 := <-uwsCh1 + s.NoError(uwsRes1.err) - s.EqualHistoryEvents(` - 1 WorkflowExecutionStarted - 2 WorkflowTaskScheduled - 3 WorkflowTaskStarted - 4 WorkflowTaskCompleted - `, s.GetHistory(s.Namespace().String(), tv.WorkflowExecution())) + // 2nd update-with-start: using *same* RequestID - but *different* UpdateID + updReq2 := updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(2), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh2 := sendUpdateWithStart(s, testcore.NewContext(), startReq, updReq2) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + uwsRes2 := <-uwsCh2 + s.NoError(uwsRes1.err) + + s.Equal(uwsRes1.response.Responses[0].GetStartWorkflow().RunId, uwsRes2.response.Responses[0].GetStartWorkflow().RunId) + }) + } }) }) - s.Run("workflow id conflict policy terminate-existing", func() { - - s.Run("terminate workflow first, then start and update", func() { - tv := testvars.New(s.T()) - - // start workflow - firstWF, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) + t.Run("workflow is closed", func(t *testing.T) { + t.Run("workflow id reuse policy allow-duplicate", func(t *testing.T) { + s := testcore.NewEnv(t) + // start and terminate workflow + initialWorkflow, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) s.NoError(err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), + &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) s.NoError(err) // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING - updateReq := updateWorkflowRequest(s, tv, + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE + updateReq := updateWorkflowRequest(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), }, nil }) s.NoError(err) @@ -5207,438 +5445,166 @@ func (s *UpdateWorkflowSuite) TestUpdateWithStart() { startResp := uwsRes.response.Responses[0].GetStartWorkflow() updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() requireStartedAndRunning(s.T(), startResp) - s.Equal(startResp.RunId, updateRep.UpdateRef.GetWorkflowExecution().RunId) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) - // ensure workflow was terminated - descResp, err := s.FrontendClient().DescribeWorkflowExecution(testcore.NewContext(), - &workflowservice.DescribeWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - Execution: &commonpb.WorkflowExecution{WorkflowId: startReq.WorkflowId, RunId: firstWF.RunId}, - }) - s.NoError(err) - s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, descResp.WorkflowExecutionInfo.Status) + // ensure terminated workflow is not locked by update-with-start + err = s.SendSignal(s.Namespace().String(), &commonpb.WorkflowExecution{ + WorkflowId: s.Tv().WorkflowID(), + RunId: initialWorkflow.RunId, + }, s.Tv().Any().String(), s.Tv().Any().Payloads(), s.Tv().Any().String()) + s.ErrorContains(err, "workflow execution already completed") // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, + pollRes, err := pollUpdate(s, s.Tv(), &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) + s.NoError(err) s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) }) - s.Run("given an accepted update, attach to it", func() { - tv := testvars.New(s.T()) + t.Run("workflow id reuse policy reject-duplicate", func(t *testing.T) { + s := testcore.NewEnv(t) + // start and terminate workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) - // 1st update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh1 := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) - // accept the update - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, + _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), + &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + uwsRes := <-uwsCh + s.Error(uwsRes.err) + s.Equal("Update-with-Start could not be executed.", uwsRes.err.Error()) + errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + s.Contains(errs[0].Error(), "Workflow execution already finished") + var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted + s.ErrorAs(errs[0], &alreadyStartedErr) + s.Equal("Operation was aborted.", errs[1].Error()) + }) + + t.Run("receive completed update result", func(t *testing.T) { + _ = testcore.NewEnv(t) // unused s + for _, p := range []enumspb.WorkflowIdConflictPolicy{ + enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + } { + t.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func(t *testing.T) { + s := testcore.NewEnv(t) + // 1st update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = p + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + Commands: s.UpdateAcceptCompleteCommands(s.Tv()), + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + startResp1 := uwsRes.response.Responses[0].GetStartWorkflow() + _ = uwsRes.response.Responses[1].GetUpdateWorkflow() + requireStartedAndRunning(s.T(), startResp1) + + // terminate workflow + _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), + &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: s.Tv().WorkflowExecution(), + Reason: s.Tv().Any().String(), + }) + s.NoError(err) + + // 2nd update-with-start (using the same Update ID but different Request ID) + uwsRes = <-sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + s.NoError(uwsRes.err) + startResp := uwsRes.response.Responses[0].GetStartWorkflow() + updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() + s.False(startResp.Started) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, startResp.Status) + // TODO: check startResp.Running + s.Equal("success-result-of-"+s.Tv().UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) + }) + } + }) + }) + + t.Run("workflow start conflict", func(t *testing.T) { + t.Run("workflow id conflict policy fail: use-existing", func(t *testing.T) { + // Uses InjectHook which requires a dedicated cluster to avoid conflicts with other tests. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) + + // simulate a race condition + s.InjectHook(testhooks.UpdateWithStartInBetweenLockAndStart, func() { + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startReq) + s.NoError(err) + }) + + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{}, nil + }) + s.NoError(err) + + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptMessages(tv, task.Messages[0]), + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), }, nil }) s.NoError(err) - uwsRes1 := <-uwsCh1 - s.NoError(uwsRes1.err) - startResp1 := uwsRes1.response.Responses[0].GetStartWorkflow() - updateRep1 := uwsRes1.response.Responses[1].GetUpdateWorkflow() - s.True(startResp1.Started) - s.Equal(startResp1.RunId, updateRep1.UpdateRef.GetWorkflowExecution().RunId) - s.Equal(updateRep1.Stage, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED) - - // 2nd update-with-start: attaches to update instead of terminating workflow - uwsCh2 := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - uwsRes2 := <-uwsCh2 - s.NoError(uwsRes2.err) - startResp2 := uwsRes2.response.Responses[0].GetStartWorkflow() - updateRep2 := uwsRes2.response.Responses[1].GetUpdateWorkflow() - s.False(startResp2.Started) - s.Equal(startResp2.RunId, startResp1.RunId) // no termination - s.Equal(updateRep2.Stage, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED) + <-uwsCh }) }) - s.Run("workflow id conflict policy fail: abort multi operation", func() { - tv := testvars.New(s.T()) - - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - - // start workflow - startWorkflowReq(tv) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL - updateReq := updateWorkflowRequest(s, tv, &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - uwsRes := <-uwsCh - s.Error(uwsRes.err) - s.Equal("Update-with-Start could not be executed.", uwsRes.err.Error()) - errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted - s.ErrorAs(errs[0], &alreadyStartedErr) - s.Equal("Operation was aborted.", errs[1].Error()) - }) - - s.Run("receive completed update result", func() { - for _, p := range []enumspb.WorkflowIdConflictPolicy{ - enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - } { - s.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func() { - tv := testvars.New(s.T()) - - startReq := startWorkflowReq(tv) - updReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - - // 1st update-with-start - uwsCh1 := sendUpdateWithStart(testcore.NewContext(), startReq, updReq) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes1 := <-uwsCh1 - s.NoError(uwsRes1.err) - - // 2nd update-with-start: using *same* UpdateID - but *different* RequestID - uwsRes2 := <-sendUpdateWithStart(testcore.NewContext(), startReq, updReq) - s.NoError(uwsRes2.err) - - s.Equal(uwsRes1.response.Responses[0].GetStartWorkflow().RunId, uwsRes2.response.Responses[0].GetStartWorkflow().RunId) - s.Equal(uwsRes1.response.Responses[1].GetUpdateWorkflow().Outcome.String(), uwsRes2.response.Responses[1].GetUpdateWorkflow().Outcome.String()) - - // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) - s.Equal(uwsRes1.response.Responses[1].GetUpdateWorkflow().Outcome.String(), pollRes.Outcome.String()) - }) - } - }) - - s.Run("dedupes start", func() { - for _, p := range []enumspb.WorkflowIdConflictPolicy{ - enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - } { - s.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func() { - tv := testvars.New(s.T()) - - startReq := startWorkflowReq(tv) - startReq.RequestId = "request_id" - startReq.WorkflowIdConflictPolicy = p - updReq1 := updateWorkflowRequest(s, tv.WithUpdateIDNumber(1), - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - - // 1st update-with-start - uwsCh1 := sendUpdateWithStart(testcore.NewContext(), startReq, updReq1) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes1 := <-uwsCh1 - s.NoError(uwsRes1.err) - - // 2nd update-with-start: using *same* RequestID - but *different* UpdateID - updReq2 := updateWorkflowRequest(s, tv.WithUpdateIDNumber(2), - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh2 := sendUpdateWithStart(testcore.NewContext(), startReq, updReq2) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes2 := <-uwsCh2 - s.NoError(uwsRes1.err) - - s.Equal(uwsRes1.response.Responses[0].GetStartWorkflow().RunId, uwsRes2.response.Responses[0].GetStartWorkflow().RunId) - }) - } - }) - }) - - s.Run("workflow is closed", func() { - - s.Run("workflow id reuse policy allow-duplicate", func() { - tv := testvars.New(s.T()) - - // start and terminate workflow - initialWorkflow, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), - &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - requireStartedAndRunning(s.T(), startResp) - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) - - // ensure terminated workflow is not locked by update-with-start - err = s.SendSignal(s.Namespace().String(), &commonpb.WorkflowExecution{ - WorkflowId: tv.WorkflowID(), - RunId: initialWorkflow.RunId, - }, tv.Any().String(), tv.Any().Payloads(), tv.Any().String()) - s.ErrorContains(err, "workflow execution already completed") - - // poll update to ensure same outcome is returned - pollRes, err := pollUpdate(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - s.Nil(err) - s.Equal(updateRep.Outcome.String(), pollRes.Outcome.String()) - }) - - s.Run("workflow id reuse policy reject-duplicate", func() { - tv := testvars.New(s.T()) - - // start and terminate workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), - &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - uwsRes := <-uwsCh - s.Error(uwsRes.err) - s.Equal("Update-with-Start could not be executed.", uwsRes.err.Error()) - errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - s.Contains(errs[0].Error(), "Workflow execution already finished") - var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted - s.ErrorAs(errs[0], &alreadyStartedErr) - s.Equal("Operation was aborted.", errs[1].Error()) - }) - - s.Run("receive completed update result", func() { - for _, p := range []enumspb.WorkflowIdConflictPolicy{ - enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, - enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - } { - s.Run(fmt.Sprintf("for workflow id conflict policy %v", p), func() { - tv := testvars.New(s.T()) - - // 1st update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = p - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - Commands: s.UpdateAcceptCompleteCommands(tv), - }, nil - }) - s.NoError(err) - - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - startResp1 := uwsRes.response.Responses[0].GetStartWorkflow() - _ = uwsRes.response.Responses[1].GetUpdateWorkflow() - requireStartedAndRunning(s.T(), startResp1) - - // terminate workflow - _, err = s.FrontendClient().TerminateWorkflowExecution(testcore.NewContext(), - &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowExecution: tv.WorkflowExecution(), - Reason: tv.Any().String(), - }) - s.NoError(err) - - // 2nd update-with-start (using the same Update ID but different Request ID) - uwsRes = <-sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - s.NoError(uwsRes.err) - startResp := uwsRes.response.Responses[0].GetStartWorkflow() - updateRep := uwsRes.response.Responses[1].GetUpdateWorkflow() - s.False(startResp.Started) - s.Equal(startResp.Status, enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED) - // TODO: check startResp.Running - s.EqualValues("success-result-of-"+tv.UpdateID(), testcore.DecodeString(s.T(), updateRep.GetOutcome().GetSuccess())) - }) - } - }) - }) - - s.Run("workflow start conflict", func() { - - s.Run("workflow id conflict policy fail: use-existing", func() { - tv := testvars.New(s.T()) - - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED}) - - // simulate a race condition - s.InjectHook(testhooks.UpdateWithStartInBetweenLockAndStart, func() { - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startReq) + t.Run("update is aborted by closing workflow", func(t *testing.T) { + t.Run("retry request once when workflow was not started", func(t *testing.T) { + s := testcore.NewEnv(t) + // start workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) s.NoError(err) - }) - - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{}, nil - }) - s.NoError(err) - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - - <-uwsCh - }) - }) - - s.Run("update is aborted by closing workflow", func() { - - s.Run("retry request once when workflow was not started", func() { - tv := testvars.New(s.T()) - - // start workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - // wait until the update is admitted - then complete workflow - waitUpdateAdmitted(s, tv) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, - }, - }, - }, - }, nil - }) - s.NoError(err) - - // update-with-start will do a server-side retry - - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - }) - - s.Run("return retryable error after retry", func() { - tv := testvars.New(s.T()) - - // start workflow - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) - s.NoError(err) - _, err = s.TaskPoller().PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) - s.NoError(err) - - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) - - // wait until the update is admitted - waitUpdateAdmitted(s, tv) - - s.InjectHook(testhooks.UpdateWithStartOnClosingWorkflowRetry, func() { - _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(tv)) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) s.NoError(err) - }) - // complete workflow (twice including retry) - for i := 0; i < 2; i++ { - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + // wait until the update is admitted - then complete workflow + waitUpdateAdmitted(s, s.Tv()) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { return &workflowservice.RespondWorkflowTaskCompletedRequest{ Commands: []*commandpb.Command{ @@ -5652,165 +5618,198 @@ func (s *UpdateWorkflowSuite) TestUpdateWithStart() { }, nil }) s.NoError(err) - } - // ensure update-with-start returns retryable error - uwsRes := <-uwsCh - s.Error(uwsRes.err) - errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - s.Equal("Operation was aborted.", errs[0].Error()) - s.ErrorContains(errs[1], update.AbortedByWorkflowClosingErr.Error()) - s.IsType(&serviceerror.Aborted{}, errs[1]) - }) + // update-with-start will do a server-side retry - s.Run("do not retry when workflow was started", func() { - tv := testvars.New(s.T()) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) - // update-with-start - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - updateReq := updateWorkflowRequest(s, tv, - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(testcore.NewContext(), startReq, updateReq) + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + }) - // wait until the update is admitted - then complete workflow - waitUpdateAdmitted(s, tv) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: []*commandpb.Command{ - { - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + t.Run("return retryable error after retry", func(t *testing.T) { + // Uses InjectHook which requires a dedicated cluster to avoid conflicts with other tests. + s := testcore.NewEnv(t, testcore.WithDedicatedCluster()) + // start workflow + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) + _, err = s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), taskpoller.DrainWorkflowTask) + s.NoError(err) + + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + // wait until the update is admitted + waitUpdateAdmitted(s, s.Tv()) + + s.InjectHook(testhooks.UpdateWithStartOnClosingWorkflowRetry, func() { + _, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowReq(s, s.Tv())) + s.NoError(err) + }) + + // complete workflow (twice including retry) + for i := 0; i < 2; i++ { + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }, + }, + }, nil + }) + s.NoError(err) + } + + // ensure update-with-start returns retryable error + uwsRes := <-uwsCh + s.Error(uwsRes.err) + errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + s.Equal("Operation was aborted.", errs[0].Error()) + s.ErrorContains(errs[1], update.AbortedByWorkflowClosingErr.Error()) + s.ErrorAs(errs[1], new(*serviceerror.Aborted)) + }) + + t.Run("do not retry when workflow was started", func(t *testing.T) { + s := testcore.NewEnv(t) + // update-with-start + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + updateReq := updateWorkflowRequest(s, s.Tv(), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, testcore.NewContext(), startReq, updateReq) + + // wait until the update is admitted - then complete workflow + waitUpdateAdmitted(s, s.Tv()) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, }, }, - }, + }, nil + }) + s.NoError(err) + + uwsRes := <-uwsCh + s.Error(uwsRes.err) + errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + s.ErrorContains(errs[1], update.AbortedByWorkflowClosingErr.Error()) + }) + }) + + t.Run("return update rate limit error", func(t *testing.T) { + // lower maximum total number of updates for testing purposes + s := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdates, 1), + ) + + ctx := testcore.NewContext() + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + + // allows 1st + updateReq := updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(0), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, ctx, startReq, updateReq) + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptCompleteMessages(s.Tv(), task.Messages[0]), }, nil }) s.NoError(err) - uwsRes := <-uwsCh - s.Error(uwsRes.err) - errs := uwsRes.err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - s.ErrorContains(errs[1], update.AbortedByWorkflowClosingErr.Error()) + s.NoError(uwsRes.err) + + // denies 2nd + updateReq = updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(1), updateReq.WaitPolicy) + select { + case <-sendUpdateWithStart(s, ctx, startReq, updateReq): + err = (<-sendUpdateWithStart(s, ctx, startReq, updateReq)).err + s.Error(err) + errs := err.(*serviceerror.MultiOperationExecution).OperationErrors() + s.Len(errs, 2) + s.Equal("Operation was aborted.", errs[0].Error()) + s.Contains(errs[1].Error(), "limit on the total number of distinct updates in this workflow has been reached") + case <-ctx.Done(): + s.Fail("timed out waiting for update") + } + }) + + t.Run("return update in-flight limit error", func(t *testing.T) { + // lower maximum in-flight updates for testing purposes + maxInFlight := 1 + s := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.WorkflowExecutionMaxInFlightUpdates, maxInFlight), + ) + + ctx := testcore.NewContext() + startReq := startWorkflowReq(s, s.Tv()) + startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + + // Start workflow and admit 1st update (but don't complete it) + updateReq := updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(0), + &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) + uwsCh := sendUpdateWithStart(s, ctx, startReq, updateReq) + + // Poll workflow task but only accept, don't complete the update + _, err := s.TaskPoller().PollAndHandleWorkflowTask(s.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Messages: s.UpdateAcceptMessages(s.Tv(), task.Messages[0]), + }, nil + }) + s.NoError(err) + uwsRes := <-uwsCh + s.NoError(uwsRes.err) + + // Try to send 2nd update-with-start while 1st is still in-flight (not completed) + updateReq = updateWorkflowRequest(s, s.Tv().WithUpdateIDNumber(1), updateReq.WaitPolicy) + uwsCh = sendUpdateWithStart(s, ctx, startReq, updateReq) + select { + case uwsRes := <-uwsCh: + err = uwsRes.err + s.Error(err) + + var multiOpsErr *serviceerror.MultiOperationExecution + s.ErrorAs(err, &multiOpsErr) + + errs := multiOpsErr.OperationErrors() + s.Len(errs, 2) + s.Equal("Operation was aborted.", errs[0].Error()) + s.Contains(errs[1].Error(), "limit on number of concurrent in-flight updates has been reached") + + // Verify ResourceExhausted error is accessible with all details preserved + var resExhausted *serviceerror.ResourceExhausted + s.ErrorAs(errs[1], &resExhausted) + s.Equal(enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, resExhausted.Cause) + s.Equal(enumspb.RESOURCE_EXHAUSTED_SCOPE_NAMESPACE, resExhausted.Scope) + s.Contains(resExhausted.Message, "limit on number of concurrent in-flight updates") + case <-ctx.Done(): + s.Fail("timed out waiting for update") + } }) }) - - s.Run("return update rate limit error", func() { - // lower maximum total number of updates for testing purposes - maxTotalUpdates := 1 - cleanup := s.OverrideDynamicConfig(dynamicconfig.WorkflowExecutionMaxTotalUpdates, maxTotalUpdates) - defer cleanup() - - ctx := testcore.NewContext() - tv := testvars.New(s.T()) - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - - // allows 1st - updateReq := updateWorkflowRequest(s, tv.WithUpdateIDNumber(0), - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(ctx, startReq, updateReq) - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptCompleteMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - - // denies 2nd - updateReq = updateWorkflowRequest(s, tv.WithUpdateIDNumber(1), updateReq.WaitPolicy) - select { - case <-sendUpdateWithStart(ctx, startReq, updateReq): - err = (<-sendUpdateWithStart(ctx, startReq, updateReq)).err - s.Error(err) - errs := err.(*serviceerror.MultiOperationExecution).OperationErrors() - s.Len(errs, 2) - s.Equal("Operation was aborted.", errs[0].Error()) - s.Contains(errs[1].Error(), "limit on the total number of distinct updates in this workflow has been reached") - case <-ctx.Done(): - s.Fail("timed out waiting for update") - } - }) - - s.Run("return update in-flight limit error", func() { - // lower maximum in-flight updates for testing purposes - maxInFlight := 1 - cleanup := s.OverrideDynamicConfig(dynamicconfig.WorkflowExecutionMaxInFlightUpdates, maxInFlight) - defer cleanup() - - ctx := testcore.NewContext() - tv := testvars.New(s.T()) - startReq := startWorkflowReq(tv) - startReq.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING - - // Start workflow and admit 1st update (but don't complete it) - updateReq := updateWorkflowRequest(s, tv.WithUpdateIDNumber(0), - &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) - uwsCh := sendUpdateWithStart(ctx, startReq, updateReq) - - // Poll workflow task but only accept, don't complete the update - _, err := s.TaskPoller().PollAndHandleWorkflowTask(tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Messages: s.UpdateAcceptMessages(tv, task.Messages[0]), - }, nil - }) - s.NoError(err) - uwsRes := <-uwsCh - s.NoError(uwsRes.err) - - // Try to send 2nd update-with-start while 1st is still in-flight (not completed) - updateReq = updateWorkflowRequest(s, tv.WithUpdateIDNumber(1), updateReq.WaitPolicy) - uwsCh = sendUpdateWithStart(ctx, startReq, updateReq) - select { - case uwsRes := <-uwsCh: - err = uwsRes.err - s.Error(err) - - var multiOpsErr *serviceerror.MultiOperationExecution - s.ErrorAs(err, &multiOpsErr) - - errs := multiOpsErr.OperationErrors() - s.Len(errs, 2) - s.Equal("Operation was aborted.", errs[0].Error()) - s.Contains(errs[1].Error(), "limit on number of concurrent in-flight updates has been reached") - - // Verify ResourceExhausted error is accessible with all details preserved - var resExhausted *serviceerror.ResourceExhausted - s.ErrorAs(errs[1], &resExhausted) - s.Equal(enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, resExhausted.Cause) - s.Equal(enumspb.RESOURCE_EXHAUSTED_SCOPE_NAMESPACE, resExhausted.Scope) - s.Contains(resExhausted.Message, "limit on number of concurrent in-flight updates") - case <-ctx.Done(): - s.Fail("timed out waiting for update") - } - }) -} - -func (s *UpdateWorkflowSuite) closeShard(wid string) { - s.T().Helper() - - resp, err := s.FrontendClient().DescribeNamespace(testcore.NewContext(), &workflowservice.DescribeNamespaceRequest{ - Namespace: s.Namespace().String(), - }) - s.NoError(err) - - _, err = s.AdminClient().CloseShard(testcore.NewContext(), &adminservice.CloseShardRequest{ - ShardId: common.WorkflowIDToHistoryShard(resp.NamespaceInfo.Id, wid, s.GetTestClusterConfig().HistoryConfig.NumHistoryShards), - }) - s.NoError(err) -} - -func (s *UpdateWorkflowSuite) useRunID(tv *testvars.TestVars, useRunID bool, runID string) *testvars.TestVars { - if useRunID { - return tv.WithRunID(runID) - } - return tv } diff --git a/tests/update_workflow_utils.go b/tests/update_workflow_utils.go index 2717461bf4..d2ac9fc54b 100644 --- a/tests/update_workflow_utils.go +++ b/tests/update_workflow_utils.go @@ -19,22 +19,22 @@ type updateResponseErr struct { err error } -func sendUpdate(ctx context.Context, s testEnv, tv *testvars.TestVars) <-chan updateResponseErr { +func sendUpdate(ctx context.Context, s testcore.Env, tv *testvars.TestVars) <-chan updateResponseErr { s.T().Helper() return sendUpdateInternal(ctx, s, tv, nil, false) } -func sendUpdateNoError(s testEnv, tv *testvars.TestVars) <-chan *workflowservice.UpdateWorkflowExecutionResponse { +func sendUpdateNoError(s testcore.Env, tv *testvars.TestVars) <-chan *workflowservice.UpdateWorkflowExecutionResponse { s.T().Helper() return sendUpdateNoErrorInternal(s, tv, nil) } -func sendUpdateNoErrorWaitPolicyAccepted(s testEnv, tv *testvars.TestVars) <-chan *workflowservice.UpdateWorkflowExecutionResponse { +func sendUpdateNoErrorWaitPolicyAccepted(s testcore.Env, tv *testvars.TestVars) <-chan *workflowservice.UpdateWorkflowExecutionResponse { s.T().Helper() return sendUpdateNoErrorInternal(s, tv, &updatepb.WaitPolicy{LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ACCEPTED}) } -func pollUpdate(s testEnv, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy) (*workflowservice.PollWorkflowExecutionUpdateResponse, error) { +func pollUpdate(s testcore.Env, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy) (*workflowservice.PollWorkflowExecutionUpdateResponse, error) { s.T().Helper() return s.FrontendClient().PollWorkflowExecutionUpdate(testcore.NewContext(), &workflowservice.PollWorkflowExecutionUpdateRequest{ Namespace: s.Namespace().String(), @@ -47,7 +47,7 @@ func pollUpdate(s testEnv, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolic } func updateWorkflowRequest( - s testEnv, + s testcore.Env, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy, ) *workflowservice.UpdateWorkflowExecutionRequest { @@ -65,7 +65,7 @@ func updateWorkflowRequest( } } -func sendUpdateNoErrorInternal(s testEnv, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy) <-chan *workflowservice.UpdateWorkflowExecutionResponse { +func sendUpdateNoErrorInternal(s testcore.Env, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy) <-chan *workflowservice.UpdateWorkflowExecutionResponse { s.T().Helper() retCh := make(chan *workflowservice.UpdateWorkflowExecutionResponse) syncCh := make(chan struct{}) @@ -80,7 +80,7 @@ func sendUpdateNoErrorInternal(s testEnv, tv *testvars.TestVars, waitPolicy *upd func sendUpdateInternal( ctx context.Context, - s testEnv, + s testcore.Env, tv *testvars.TestVars, waitPolicy *updatepb.WaitPolicy, requireNoError bool, @@ -99,7 +99,7 @@ func sendUpdateInternal( return updateResultCh } -func waitUpdateAdmitted(s testEnv, tv *testvars.TestVars) { +func waitUpdateAdmitted(s testcore.Env, tv *testvars.TestVars) { s.T().Helper() require.EventuallyWithTf(s.T(), func(collect *assert.CollectT) { pollResp, pollErr := s.FrontendClient().PollWorkflowExecutionUpdate(testcore.NewContext(), &workflowservice.PollWorkflowExecutionUpdateRequest{ diff --git a/tests/workflow_utils.go b/tests/workflow_utils.go index ca84a71aec..2f8f351540 100644 --- a/tests/workflow_utils.go +++ b/tests/workflow_utils.go @@ -1,21 +1,12 @@ package tests import ( - "testing" - "go.temporal.io/api/workflowservice/v1" - "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/tests/testcore" ) -type testEnv interface { - T() *testing.T - Namespace() namespace.Name - FrontendClient() workflowservice.WorkflowServiceClient -} - -func mustStartWorkflow(s testEnv, tv *testvars.TestVars) string { +func mustStartWorkflow(s testcore.Env, tv *testvars.TestVars) string { s.T().Helper() startResp, err := s.FrontendClient().StartWorkflowExecution(testcore.NewContext(), startWorkflowRequest(s, tv)) if err != nil { @@ -24,7 +15,7 @@ func mustStartWorkflow(s testEnv, tv *testvars.TestVars) string { return startResp.GetRunId() } -func startWorkflowRequest(s testEnv, tv *testvars.TestVars) *workflowservice.StartWorkflowExecutionRequest { +func startWorkflowRequest(s testcore.Env, tv *testvars.TestVars) *workflowservice.StartWorkflowExecutionRequest { return &workflowservice.StartWorkflowExecutionRequest{ RequestId: tv.Any().String(), Namespace: s.Namespace().String(), diff --git a/tests/xdc/base.go b/tests/xdc/base.go index f0568fa9ec..c5a0846080 100644 --- a/tests/xdc/base.go +++ b/tests/xdc/base.go @@ -101,6 +101,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { s.dynamicConfigOverrides[dynamicconfig.VisibilityProcessorMaxPollInterval.Key()] = time.Second * 3 s.dynamicConfigOverrides[dynamicconfig.OutboundProcessorMaxPollInterval.Key()] = time.Second * 3 + persistenceDefaults := testcore.GetPersistenceTestDefaults() clusterConfigs := []*testcore.TestClusterConfig{ { ClusterMetadata: cluster.Config{ @@ -110,6 +111,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { HistoryConfig: testcore.HistoryConfig{ NumHistoryShards: cmp.Or(params.NumHistoryShards, 1), }, + Persistence: persistenceDefaults, }, { ClusterMetadata: cluster.Config{ @@ -119,6 +121,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { HistoryConfig: testcore.HistoryConfig{ NumHistoryShards: cmp.Or(params.NumHistoryShards, 1), }, + Persistence: persistenceDefaults, }, } @@ -131,7 +134,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { clusterConfigs[clusterIndex].ClusterMetadata.MasterClusterName = clusterName clusterConfigs[clusterIndex].ClusterMetadata.CurrentClusterName = clusterName clusterConfigs[clusterIndex].ClusterMetadata.EnableGlobalNamespace = true - clusterConfigs[clusterIndex].Persistence.DBName = "func_tests_" + clusterName + clusterConfigs[clusterIndex].Persistence.DBName += "_" + clusterName clusterConfigs[clusterIndex].ClusterMetadata.ClusterInformation = map[string]cluster.ClusterInformation{ clusterName: { Enabled: true, diff --git a/tests/xdc/replication_enable_test.go b/tests/xdc/replication_enable_test.go index 4f807ef7ee..4aea135f82 100644 --- a/tests/xdc/replication_enable_test.go +++ b/tests/xdc/replication_enable_test.go @@ -51,6 +51,7 @@ func (s *ReplicationEnableTestSuite) SetupSuite() { dynamicconfig.SendRawHistoryBetweenInternalServices.Key(): true, } + persistenceDefaults := testcore.GetPersistenceTestDefaults() clusterConfigs := []*testcore.TestClusterConfig{ { ClusterMetadata: cluster.Config{ @@ -60,6 +61,7 @@ func (s *ReplicationEnableTestSuite) SetupSuite() { HistoryConfig: testcore.HistoryConfig{ NumHistoryShards: 1, }, + Persistence: persistenceDefaults, }, { ClusterMetadata: cluster.Config{ @@ -69,6 +71,7 @@ func (s *ReplicationEnableTestSuite) SetupSuite() { HistoryConfig: testcore.HistoryConfig{ NumHistoryShards: 1, }, + Persistence: persistenceDefaults, }, } @@ -82,7 +85,7 @@ func (s *ReplicationEnableTestSuite) SetupSuite() { clusterConfigs[clusterIndex].ClusterMetadata.MasterClusterName = clusterName clusterConfigs[clusterIndex].ClusterMetadata.CurrentClusterName = clusterName clusterConfigs[clusterIndex].ClusterMetadata.EnableGlobalNamespace = true - clusterConfigs[clusterIndex].Persistence.DBName = "func_tests_" + clusterName + clusterConfigs[clusterIndex].Persistence.DBName += "_" + clusterName clusterConfigs[clusterIndex].ClusterMetadata.ClusterInformation = map[string]cluster.ClusterInformation{ clusterName: { Enabled: true,