Add chasm task type dynamic config filter and standby task discard delay dynamic config flag (#9506)

## What changed?
Add chasm logical task type dynamic config filter and standby task
discard delay dynamic config flag.

## Why?
Standby tasks have a configurable dynamic config (per task type) that
specifies the timeout for discarding the task and running the
post-discard function. Currently, this is not configurable for CHASM
logical tasks. By default, CHASM tasks need a higher discard timeout
since they are not regenerated from pending tasks in MS, which can lead
to abandoned tasks. For tasks that can be safely dispatched to Matching,
they can be configured with a lower value. (See Standalone activity
tasks).

## How did you test it?
- [X] built
- [X] run locally and tested manually
- [X] covered by existing tests
- [ ] added new unit test(s)
- [ ] added new functional test(s)
This commit is contained in:
Alan Wu
2026-03-13 00:57:57 -04:00
committed by GitHub
parent b6ae24e69e
commit 5845841278
11 changed files with 291 additions and 2 deletions

View File

@@ -110,6 +110,11 @@ var (
{},
}`,
},
{
Name: "ChasmTaskType",
GoArgs: "chasmTaskType string",
Expr: "[]Constraints{{ChasmTaskType: chasmTaskType}, {}}",
},
}}
)

View File

@@ -89,6 +89,7 @@ type (
NamespaceID string
TaskQueueName string
Destination string
ChasmTaskType string
TaskQueueType enumspb.TaskQueueType
ShardID int32
TaskType enumsspb.TaskType

View File

@@ -41,6 +41,7 @@ const (
testGetStringPropertyFilteredByNamespaceKey = "testGetStringPropertyFilteredByNamespaceKey"
testGetStringPropertyFilteredByNamespaceIDKey = "testGetStringPropertyFilteredByNamespaceIDKey"
testGetIntPropertyFilteredByDestinationKey = "testGetIntPropertyFilteredByDestinationKey"
testGetDurationPropertyFilteredByChasmTaskTypeKey = "testGetDurationPropertyFilteredByChasmTaskTypeKey"
)
// Note: fileBasedClientSuite also heavily tests Collection, since some tests are easier with data
@@ -200,6 +201,15 @@ func (s *collectionSuite) TestGetDurationPropertyFilteredByTaskType() {
s.Equal(time.Minute, value(taskType))
}
func (s *collectionSuite) TestGetDurationPropertyFilteredByChasmTaskType() {
setting := dynamicconfig.NewChasmTaskTypeDurationSetting(testGetDurationPropertyFilteredByChasmTaskTypeKey, time.Second, "")
chasmTaskType := "activity.dispatch"
value := setting.Get(s.cln)
s.Equal(time.Second, value(chasmTaskType))
s.client.SetValue(testGetDurationPropertyFilteredByChasmTaskTypeKey, time.Minute)
s.Equal(time.Minute, value(chasmTaskType))
}
func (s *collectionSuite) TestGetDurationPropertyStructuredDefaults() {
setting := dynamicconfig.NewTaskQueueDurationSettingWithConstrainedDefault(
testGetDurationPropertyStructuredDefaults,

View File

@@ -91,6 +91,12 @@ testGetDurationPropertyFilteredByTaskTypeKey:
- value: 10s
constraints:
historytasktype: 1
testGetDurationPropertyFilteredByChasmTaskTypeKey:
- value: 30s
constraints:
chasmtasktype: "activity.dispatch"
- value: 24h
constraints: {}
testGetIntPropertyFilteredByDestinationKey:
- value: 10
constraints: {}

View File

@@ -1713,6 +1713,15 @@ before calling remote for missing events`,
15*time.Minute,
`StandbyTaskMissingEventsDiscardDelay is the amount of time standby cluster's will wait (if events are missing)
before discarding the task`,
)
ChasmStandbyTaskDiscardDelay = NewChasmTaskTypeDurationSetting(
"history.ChasmStandbyTaskDiscardDelay",
24*time.Hour,
`ChasmStandbyTaskDiscardDelay is the amount of time standby cluster will wait
before discarding a CHASM task. Configurable per RegistrableTask type (e.g. "activity.dispatch").
The default is intentionally much higher than the non CHASM standby discard delay because
discarding a CHASM task can leave the execution in a stuck state after failover. Task types
that can be safely offloaded should be configured with a shorter delay.`,
)
QueuePendingTaskCriticalCount = NewGlobalIntSetting(
"history.queuePendingTaskCriticalCount",

View File

@@ -272,6 +272,14 @@ func (s *fileBasedClientSuite) TestGetDurationValue_FilteredByTaskTypeQueue() {
s.Equal(expectedValue, v)
}
func (s *fileBasedClientSuite) TestGetDurationValue_FilteredByChasmTaskType() {
setting := dynamicconfig.NewChasmTaskTypeDurationSetting(testGetDurationPropertyFilteredByChasmTaskTypeKey, 0, "")
v := setting.Get(s.collection)("activity.dispatch")
s.Equal(30*time.Second, v)
v = setting.Get(s.collection)("callback.invoke")
s.Equal(24*time.Hour, v)
}
func (s *fileBasedClientSuite) TestValidateConfig_NilLogger() {
doneCh := make(chan any)
defer close(doneCh)

View File

@@ -19,6 +19,7 @@ const (
PrecedenceShardID
PrecedenceTaskType
PrecedenceDestination
PrecedenceChasmTaskType
)
type GlobalBoolSetting = GlobalTypedSetting[bool]
@@ -140,6 +141,23 @@ func GetBoolPropertyFnFilteredByDestination(value bool) BoolPropertyFnWithDestin
return GetTypedPropertyFnFilteredByDestination(value)
}
type ChasmTaskTypeBoolSetting = ChasmTaskTypeTypedSetting[bool]
type ChasmTaskTypeBoolConstrainedDefaultSetting = ChasmTaskTypeTypedConstrainedDefaultSetting[bool]
func NewChasmTaskTypeBoolSetting(key string, def bool, description string) ChasmTaskTypeBoolSetting {
return NewChasmTaskTypeTypedSettingWithConverter[bool](key, convertBool, def, description)
}
func NewChasmTaskTypeBoolSettingWithConstrainedDefault(key string, cdef []TypedConstrainedValue[bool], description string) ChasmTaskTypeBoolConstrainedDefaultSetting {
return NewChasmTaskTypeTypedSettingWithConstrainedDefault[bool](key, convertBool, cdef, description)
}
type BoolPropertyFnWithChasmTaskTypeFilter = TypedPropertyFnWithChasmTaskTypeFilter[bool]
func GetBoolPropertyFnFilteredByChasmTaskType(value bool) BoolPropertyFnWithChasmTaskTypeFilter {
return GetTypedPropertyFnFilteredByChasmTaskType(value)
}
type GlobalIntSetting = GlobalTypedSetting[int]
type GlobalIntConstrainedDefaultSetting = GlobalTypedConstrainedDefaultSetting[int]
@@ -259,6 +277,23 @@ func GetIntPropertyFnFilteredByDestination(value int) IntPropertyFnWithDestinati
return GetTypedPropertyFnFilteredByDestination(value)
}
type ChasmTaskTypeIntSetting = ChasmTaskTypeTypedSetting[int]
type ChasmTaskTypeIntConstrainedDefaultSetting = ChasmTaskTypeTypedConstrainedDefaultSetting[int]
func NewChasmTaskTypeIntSetting(key string, def int, description string) ChasmTaskTypeIntSetting {
return NewChasmTaskTypeTypedSettingWithConverter[int](key, convertInt, def, description)
}
func NewChasmTaskTypeIntSettingWithConstrainedDefault(key string, cdef []TypedConstrainedValue[int], description string) ChasmTaskTypeIntConstrainedDefaultSetting {
return NewChasmTaskTypeTypedSettingWithConstrainedDefault[int](key, convertInt, cdef, description)
}
type IntPropertyFnWithChasmTaskTypeFilter = TypedPropertyFnWithChasmTaskTypeFilter[int]
func GetIntPropertyFnFilteredByChasmTaskType(value int) IntPropertyFnWithChasmTaskTypeFilter {
return GetTypedPropertyFnFilteredByChasmTaskType(value)
}
type GlobalFloatSetting = GlobalTypedSetting[float64]
type GlobalFloatConstrainedDefaultSetting = GlobalTypedConstrainedDefaultSetting[float64]
@@ -378,6 +413,23 @@ func GetFloatPropertyFnFilteredByDestination(value float64) FloatPropertyFnWithD
return GetTypedPropertyFnFilteredByDestination(value)
}
type ChasmTaskTypeFloatSetting = ChasmTaskTypeTypedSetting[float64]
type ChasmTaskTypeFloatConstrainedDefaultSetting = ChasmTaskTypeTypedConstrainedDefaultSetting[float64]
func NewChasmTaskTypeFloatSetting(key string, def float64, description string) ChasmTaskTypeFloatSetting {
return NewChasmTaskTypeTypedSettingWithConverter[float64](key, convertFloat, def, description)
}
func NewChasmTaskTypeFloatSettingWithConstrainedDefault(key string, cdef []TypedConstrainedValue[float64], description string) ChasmTaskTypeFloatConstrainedDefaultSetting {
return NewChasmTaskTypeTypedSettingWithConstrainedDefault[float64](key, convertFloat, cdef, description)
}
type FloatPropertyFnWithChasmTaskTypeFilter = TypedPropertyFnWithChasmTaskTypeFilter[float64]
func GetFloatPropertyFnFilteredByChasmTaskType(value float64) FloatPropertyFnWithChasmTaskTypeFilter {
return GetTypedPropertyFnFilteredByChasmTaskType(value)
}
type GlobalStringSetting = GlobalTypedSetting[string]
type GlobalStringConstrainedDefaultSetting = GlobalTypedConstrainedDefaultSetting[string]
@@ -497,6 +549,23 @@ func GetStringPropertyFnFilteredByDestination(value string) StringPropertyFnWith
return GetTypedPropertyFnFilteredByDestination(value)
}
type ChasmTaskTypeStringSetting = ChasmTaskTypeTypedSetting[string]
type ChasmTaskTypeStringConstrainedDefaultSetting = ChasmTaskTypeTypedConstrainedDefaultSetting[string]
func NewChasmTaskTypeStringSetting(key string, def string, description string) ChasmTaskTypeStringSetting {
return NewChasmTaskTypeTypedSettingWithConverter[string](key, convertString, def, description)
}
func NewChasmTaskTypeStringSettingWithConstrainedDefault(key string, cdef []TypedConstrainedValue[string], description string) ChasmTaskTypeStringConstrainedDefaultSetting {
return NewChasmTaskTypeTypedSettingWithConstrainedDefault[string](key, convertString, cdef, description)
}
type StringPropertyFnWithChasmTaskTypeFilter = TypedPropertyFnWithChasmTaskTypeFilter[string]
func GetStringPropertyFnFilteredByChasmTaskType(value string) StringPropertyFnWithChasmTaskTypeFilter {
return GetTypedPropertyFnFilteredByChasmTaskType(value)
}
type GlobalDurationSetting = GlobalTypedSetting[time.Duration]
type GlobalDurationConstrainedDefaultSetting = GlobalTypedConstrainedDefaultSetting[time.Duration]
@@ -616,6 +685,23 @@ func GetDurationPropertyFnFilteredByDestination(value time.Duration) DurationPro
return GetTypedPropertyFnFilteredByDestination(value)
}
type ChasmTaskTypeDurationSetting = ChasmTaskTypeTypedSetting[time.Duration]
type ChasmTaskTypeDurationConstrainedDefaultSetting = ChasmTaskTypeTypedConstrainedDefaultSetting[time.Duration]
func NewChasmTaskTypeDurationSetting(key string, def time.Duration, description string) ChasmTaskTypeDurationSetting {
return NewChasmTaskTypeTypedSettingWithConverter[time.Duration](key, convertDuration, def, description)
}
func NewChasmTaskTypeDurationSettingWithConstrainedDefault(key string, cdef []TypedConstrainedValue[time.Duration], description string) ChasmTaskTypeDurationConstrainedDefaultSetting {
return NewChasmTaskTypeTypedSettingWithConstrainedDefault[time.Duration](key, convertDuration, cdef, description)
}
type DurationPropertyFnWithChasmTaskTypeFilter = TypedPropertyFnWithChasmTaskTypeFilter[time.Duration]
func GetDurationPropertyFnFilteredByChasmTaskType(value time.Duration) DurationPropertyFnWithChasmTaskTypeFilter {
return GetTypedPropertyFnFilteredByChasmTaskType(value)
}
type GlobalMapSetting = GlobalTypedSetting[map[string]any]
type GlobalMapConstrainedDefaultSetting = GlobalTypedConstrainedDefaultSetting[map[string]any]
@@ -735,6 +821,23 @@ func GetMapPropertyFnFilteredByDestination(value map[string]any) MapPropertyFnWi
return GetTypedPropertyFnFilteredByDestination(value)
}
type ChasmTaskTypeMapSetting = ChasmTaskTypeTypedSetting[map[string]any]
type ChasmTaskTypeMapConstrainedDefaultSetting = ChasmTaskTypeTypedConstrainedDefaultSetting[map[string]any]
func NewChasmTaskTypeMapSetting(key string, def map[string]any, description string) ChasmTaskTypeMapSetting {
return NewChasmTaskTypeTypedSettingWithConverter[map[string]any](key, convertMap, def, description)
}
func NewChasmTaskTypeMapSettingWithConstrainedDefault(key string, cdef []TypedConstrainedValue[map[string]any], description string) ChasmTaskTypeMapConstrainedDefaultSetting {
return NewChasmTaskTypeTypedSettingWithConstrainedDefault[map[string]any](key, convertMap, cdef, description)
}
type MapPropertyFnWithChasmTaskTypeFilter = TypedPropertyFnWithChasmTaskTypeFilter[map[string]any]
func GetMapPropertyFnFilteredByChasmTaskType(value map[string]any) MapPropertyFnWithChasmTaskTypeFilter {
return GetTypedPropertyFnFilteredByChasmTaskType(value)
}
type GlobalTypedSetting[T any] setting[T, func()]
type GlobalTypedConstrainedDefaultSetting[T any] constrainedDefaultSetting[T, func()]
@@ -1731,3 +1834,139 @@ func GetTypedPropertyFnFilteredByDestination[T any](value T) TypedPropertyFnWith
}
}
type ChasmTaskTypeTypedSetting[T any] setting[T, func(chasmTaskType string)]
type ChasmTaskTypeTypedConstrainedDefaultSetting[T any] constrainedDefaultSetting[T, func(chasmTaskType string)]
// NewChasmTaskTypeTypedSetting creates a setting that uses mapstructure to handle complex structured
// values. The value from dynamic config will be _merged_ over a deep copy of 'def'. Be very careful
// when using non-empty maps or slices as defaults, the result may not be what you want.
func NewChasmTaskTypeTypedSetting[T any](key string, def T, description string) ChasmTaskTypeTypedSetting[T] {
// Warn on any shared structure used with ConvertStructure, even though we handle it by deep copying.
warnDefaultSharedStructure(key, def)
// If even deep copy won't even work, we should panic early. Do that by calling deep copy once here.
_ = deepCopyForMapstructure(def)
s := ChasmTaskTypeTypedSetting[T]{
key: MakeKey(key),
def: def,
convert: ConvertStructure[T](def),
description: description,
}
register(s)
return s
}
// NewChasmTaskTypeTypedSettingWithConverter creates a setting with a custom converter function.
func NewChasmTaskTypeTypedSettingWithConverter[T any](key string, convert func(any) (T, error), def T, description string) ChasmTaskTypeTypedSetting[T] {
s := ChasmTaskTypeTypedSetting[T]{
key: MakeKey(key),
def: def,
convert: convert,
description: description,
}
register(s)
return s
}
// NewChasmTaskTypeTypedSettingWithConstrainedDefault creates a setting with a compound default value.
func NewChasmTaskTypeTypedSettingWithConstrainedDefault[T any](key string, convert func(any) (T, error), cdef []TypedConstrainedValue[T], description string) ChasmTaskTypeTypedConstrainedDefaultSetting[T] {
s := ChasmTaskTypeTypedConstrainedDefaultSetting[T]{
key: MakeKey(key),
cdef: cdef,
convert: convert,
description: description,
}
register(s)
return s
}
func (s ChasmTaskTypeTypedSetting[T]) Key() Key { return s.key }
func (s ChasmTaskTypeTypedSetting[T]) Precedence() Precedence { return PrecedenceChasmTaskType }
func (s ChasmTaskTypeTypedSetting[T]) Validate(v any) error {
_, err := s.convert(v)
return err
}
func (s ChasmTaskTypeTypedConstrainedDefaultSetting[T]) Key() Key { return s.key }
func (s ChasmTaskTypeTypedConstrainedDefaultSetting[T]) Precedence() Precedence { return PrecedenceChasmTaskType }
func (s ChasmTaskTypeTypedConstrainedDefaultSetting[T]) Validate(v any) error {
_, err := s.convert(v)
return err
}
func (s ChasmTaskTypeTypedSetting[T]) WithDefault(v T) ChasmTaskTypeTypedSetting[T] {
newS := s
newS.def = v
return newS
}
type TypedPropertyFnWithChasmTaskTypeFilter[T any] func(chasmTaskType string) T
func (s ChasmTaskTypeTypedSetting[T]) Get(c *Collection) TypedPropertyFnWithChasmTaskTypeFilter[T] {
return func(chasmTaskType string) T {
prec := []Constraints{{ChasmTaskType: chasmTaskType}, {}}
return matchAndConvert(
c,
s.key,
s.def,
s.convert,
prec,
)
}
}
func (s ChasmTaskTypeTypedConstrainedDefaultSetting[T]) Get(c *Collection) TypedPropertyFnWithChasmTaskTypeFilter[T] {
return func(chasmTaskType string) T {
prec := []Constraints{{ChasmTaskType: chasmTaskType}, {}}
return matchAndConvertWithConstrainedDefault(
c,
s.key,
s.cdef,
s.convert,
prec,
)
}
}
type TypedSubscribableWithChasmTaskTypeFilter[T any] func(chasmTaskType string, callback func(T)) (v T, cancel func())
func (s ChasmTaskTypeTypedSetting[T]) Subscribe(c *Collection) TypedSubscribableWithChasmTaskTypeFilter[T] {
return func(chasmTaskType string, callback func(T)) (T, func()) {
prec := []Constraints{{ChasmTaskType: chasmTaskType}, {}}
return subscribe(c, s.key, s.def, s.convert, prec, callback)
}
}
func (s ChasmTaskTypeTypedSetting[T]) dispatchUpdate(c *Collection, sub any, cvs []ConstrainedValue) {
dispatchUpdate(
c,
s.key,
s.convert,
sub.(*subscription[T]),
cvs,
)
}
func (s ChasmTaskTypeTypedConstrainedDefaultSetting[T]) Subscribe(c *Collection) TypedSubscribableWithChasmTaskTypeFilter[T] {
return func(chasmTaskType string, callback func(T)) (T, func()) {
prec := []Constraints{{ChasmTaskType: chasmTaskType}, {}}
return subscribeWithConstrainedDefault(c, s.key, s.cdef, s.convert, prec, callback)
}
}
func (s ChasmTaskTypeTypedConstrainedDefaultSetting[T]) dispatchUpdate(c *Collection, sub any, cvs []ConstrainedValue) {
dispatchUpdateWithConstrainedDefault(
c,
s.key,
s.convert,
sub.(*subscription[T]),
cvs,
)
}
func GetTypedPropertyFnFilteredByChasmTaskType[T any](value T) TypedPropertyFnWithChasmTaskTypeFilter[T] {
return func(chasmTaskType string) T {
return value
}
}

View File

@@ -241,6 +241,13 @@ func convertYamlConstraints(key Key, m map[string]any, precedence Precedence, lr
lr.errorf("destination constraint must be string")
}
validConstraint = precedence == PrecedenceDestination
case "chasmtasktype":
if v, ok := v.(string); ok {
cs.ChasmTaskType = v
} else {
lr.errorf("chasmtasktype constraint must be string")
}
validConstraint = precedence == PrecedenceChasmTaskType
default:
lr.errorf("unknown constraint type %q", k)
}

View File

@@ -96,6 +96,7 @@ type Config struct {
StandbyClusterDelay dynamicconfig.DurationPropertyFn
StandbyTaskMissingEventsResendDelay dynamicconfig.DurationPropertyFnWithTaskTypeFilter
StandbyTaskMissingEventsDiscardDelay dynamicconfig.DurationPropertyFnWithTaskTypeFilter
ChasmStandbyTaskDiscardDelay dynamicconfig.DurationPropertyFnWithChasmTaskTypeFilter
QueuePendingTaskCriticalCount dynamicconfig.IntPropertyFn
QueueReaderStuckCriticalAttempts dynamicconfig.IntPropertyFn
@@ -508,6 +509,7 @@ func NewConfig(
StandbyClusterDelay: dynamicconfig.StandbyClusterDelay.Get(dc),
StandbyTaskMissingEventsResendDelay: dynamicconfig.StandbyTaskMissingEventsResendDelay.Get(dc),
StandbyTaskMissingEventsDiscardDelay: dynamicconfig.StandbyTaskMissingEventsDiscardDelay.Get(dc),
ChasmStandbyTaskDiscardDelay: dynamicconfig.ChasmStandbyTaskDiscardDelay.Get(dc),
QueuePendingTaskCriticalCount: dynamicconfig.QueuePendingTaskCriticalCount.Get(dc),
QueueReaderStuckCriticalAttempts: dynamicconfig.QueueReaderStuckCriticalAttempts.Get(dc),

View File

@@ -185,6 +185,7 @@ func (t *timerQueueStandbyTaskExecutor) executeChasmSideEffectTimerTask(
)
}
chasmTaskType, _ := t.shardContext.ChasmRegistry().TaskFqnByID(task.Info.GetTypeId())
return t.processTimer(
ctx,
task,
@@ -192,7 +193,7 @@ func (t *timerQueueStandbyTaskExecutor) executeChasmSideEffectTimerTask(
getStandbyPostActionFn(
task,
t.getCurrentTime,
t.config.StandbyTaskMissingEventsDiscardDelay(task.GetType()),
t.config.ChasmStandbyTaskDiscardDelay(chasmTaskType),
t.checkExecutionStillExistsOnSourceBeforeDiscard,
),
)

View File

@@ -136,6 +136,7 @@ func (t *transferQueueStandbyTaskExecutor) executeChasmSideEffectTransferTask(
)
}
chasmTaskType, _ := t.shardContext.ChasmRegistry().TaskFqnByID(task.Info.GetTypeId())
return t.processTransfer(
ctx,
true,
@@ -144,7 +145,7 @@ func (t *transferQueueStandbyTaskExecutor) executeChasmSideEffectTransferTask(
getStandbyPostActionFn(
task,
t.getCurrentTime,
t.config.StandbyTaskMissingEventsDiscardDelay(task.GetType()),
t.config.ChasmStandbyTaskDiscardDelay(chasmTaskType),
t.checkExecutionStillExistsOnSourceBeforeDiscard,
),
)