Delete Namespace: ability to update sleep duration (#7020)

## What changed?
<!-- Describe what has changed in this PR -->
Delete Namespace: ability to update sleep duration.

## Why?
<!-- Tell your future self why have you made these changes -->
SRE experience improvement. Sometimes namespace delete delay is set to
the large duration by mistake and there was no way to change it after WF
is started.

Use:
```bash
temporal workflow update execute --namespace temporal-system --name update_namespace_delete_delay --workflow-id temporal-sys-reclaim-namespace-resources-workflow/default-deleted-93f5e --input '"10h"'
```
to update delay to the new value (10 hours, in the example above) or:
```bash
temporal workflow update execute --namespace temporal-system --name update_namespace_delete_delay --workflow-id temporal-sys-reclaim-namespace-resources-workflow/default-deleted-93f5e --input '"0"'
```
to remove delay completely.

Please note that new delay is started from the moment it is set, not
from the moment when original timer was created (i.e. if WF already
slept for 2 hours, and timer is updated to `10h`, it will sleep another
10 hours, not 8).

Access to `temporal-system` namespace is required. `WorkflowId` is built
as
`temporal-sys-reclaim-namespace-resources-workflow/<deleted_temp_name>`,
where `<deleted_temp_name>` is
`<original_name>-deleted-<first_5_ns_uuid_chars>`.

## How did you test it?
<!-- How have you verified this change? Tested locally? Added a unit
test? Checked in staging env? -->
Added new unit tests + manual run.

## Potential risks
<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->
No risks.

## Documentation
<!-- Have you made sure this change doesn't falsify anything currently
stated in `docs/`? If significant
new behavior is added, have you described that in `docs/`? -->
No.

## Is hotfix candidate?
<!-- Is this PR a hotfix candidate or does it require a notification to
be sent to the broader community? (Yes/No) -->
No.

service/worker/deletenamespace/errors/errors.go

@@ -41,7 +41,7 @@ const (
 var (
 	ErrUnableToExecuteActivity      = errors.New("unable to execute activity")
 	ErrUnableToExecuteChildWorkflow = errors.New("unable to execute child workflow")
-	ErrUnableToSleep                = errors.New("unable to sleep")
+	ErrUnableToSetUpdateHandler     = errors.New("unable to set Update handler")
 )
 
 func NewExecutionsStillExistError(count int) error {

service/worker/deletenamespace/reclaimresources/workflow.go

@@ -135,11 +135,56 @@ func ReclaimResourcesWorkflow(ctx workflow.Context, params ReclaimResourcesParam
 
 	ctx = workflow.WithTaskQueue(ctx, primitives.DeleteNamespaceActivityTQ)
 
+	var (
+		namespaceDeleteDelay = params.NamespaceDeleteDelay
+		cancelDeleteDelay    workflow.CancelFunc
+	)
+	err := workflow.SetUpdateHandlerWithOptions(ctx, "update_namespace_delete_delay", func(ctx workflow.Context, newNamespaceDeleteDelayStr string) (string, error) {
+		// This must succeed because Update validator already validated the input.
+		namespaceDeleteDelay, _ = time.ParseDuration(newNamespaceDeleteDelayStr)
+
+		var updateResult string
+		if namespaceDeleteDelay == 0 {
+			logger.Info("Namespace delete delay is removed. Namespace will be deleted immediately after all workflow executions are deleted.")
+			updateResult = "Namespace delete delay is removed."
+		} else {
+			logger.Info("Namespace delete delay is updated.", "new-delete-delay", namespaceDeleteDelay)
+			updateResult = fmt.Sprintf("Namespace delete delay is updated to %s.", namespaceDeleteDelay)
+		}
+
+		if cancelDeleteDelay != nil {
+			cancelDeleteDelay()
+			logger.Info("Existing namespace delete delay timer is cancelled.")
+			updateResult = "Existing namespace delete delay timer is cancelled. " + updateResult
+		}
+		return updateResult, nil
+	}, workflow.UpdateHandlerOptions{
+		Validator: func(_ workflow.Context, newNamespaceDeleteDelayStr string) error {
+			if newNamespaceDeleteDelayStr == "" {
+				return temporal.NewNonRetryableApplicationError("delay duration is required", errors.ValidationErrorErrType, nil)
+			}
+			newDuration, err := time.ParseDuration(newNamespaceDeleteDelayStr)
+			if err != nil {
+				return temporal.NewNonRetryableApplicationError("unable to parse delay duration", errors.ValidationErrorErrType, err)
+			}
+			if newDuration < 0 {
+				return temporal.NewNonRetryableApplicationError("delay duration must be positive", errors.ValidationErrorErrType, nil)
+			}
+			if newDuration > 30*24*time.Hour {
+				return temporal.NewNonRetryableApplicationError("delay duration must be less than 30 days", errors.ValidationErrorErrType, nil)
+			}
+			return nil
+		},
+	})
+	if err != nil {
+		return result, fmt.Errorf("%w: %v", errors.ErrUnableToSetUpdateHandler, err)
+	}
+
 	var la *LocalActivities
 
 	// Step 0. This workflow is started right after the namespace is marked as DELETED and renamed.
 	// Wait for namespace cache refresh to make sure no new executions are created.
-	err := workflow.Sleep(ctx, namespaceCacheRefreshDelay)
+	err = workflow.Sleep(ctx, namespaceCacheRefreshDelay)
 	if err != nil {
 		return result, err
 	}
@@ -151,9 +196,15 @@ func ReclaimResourcesWorkflow(ctx workflow.Context, params ReclaimResourcesParam
 	}
 
 	// Step 2. Sleep before deleting namespace from a database.
-	err = workflow.Sleep(ctx, params.NamespaceDeleteDelay)
-	if err != nil {
-		return result, fmt.Errorf("%w: %v", errors.ErrUnableToSleep, err)
+	for namespaceDeleteDelay > 0 {
+		var cancelableCtx workflow.Context
+		cancelableCtx, cancelDeleteDelay = workflow.WithCancel(ctx)
+		logger.Info("Delaying namespace delete. Send 'update_namespace_delete_delay' update to change or clear the delay.",
+			"duration", namespaceDeleteDelay.String())
+
+		ndd := namespaceDeleteDelay
+		namespaceDeleteDelay = 0
+		_ = workflow.Sleep(cancelableCtx, ndd)
 	}
 
 	// Step 3. Delete namespace from database.

service/worker/deletenamespace/reclaimresources/workflow_test.go

@@ -357,3 +357,77 @@ func Test_ReclaimResourcesWorkflow_NoActivityMocks_NoProgressMade(t *testing.T)
 	require.True(t, stderrors.As(err, &appErr))
 	require.Equal(t, errors.NoProgressErrType, appErr.Type())
 }
+
+func Test_ReclaimResourcesWorkflow_UpdateDeleteDelay(t *testing.T) {
+	testSuite := &testsuite.WorkflowTestSuite{}
+	testSuite.SetLogger(log.NewSdkLogger(log.NewTestLogger()))
+	env := testSuite.NewTestWorkflowEnvironment()
+
+	var a *Activities
+	var la *LocalActivities
+
+	env.RegisterWorkflow(deleteexecutions.DeleteExecutionsWorkflow)
+	env.OnWorkflow(deleteexecutions.DeleteExecutionsWorkflow, mock.Anything, mock.Anything).Return(deleteexecutions.DeleteExecutionsResult{
+		SuccessCount: 10,
+		ErrorCount:   0,
+	}, nil).Once()
+
+	env.OnActivity(la.CountExecutionsAdvVisibilityActivity, mock.Anything, namespace.ID("namespace-id"), namespace.Name("namespace")).Return(int64(10), nil).Once()
+	env.OnActivity(a.EnsureNoExecutionsAdvVisibilityActivity, mock.Anything, namespace.ID("namespace-id"), namespace.Name("namespace"), 0).Return(nil).Once()
+
+	env.OnActivity(la.DeleteNamespaceActivity, mock.Anything, namespace.ID("namespace-id"), namespace.Name("namespace")).Return(nil).Once()
+
+	timerNo := 0
+	env.SetOnTimerScheduledListener(func(_ string, delayDuration time.Duration) {
+		timerNo++
+		// There are 2 timers in WF. Test needs to skip the first one.
+		if timerNo == 2 {
+			require.Equal(t, 10*time.Hour, delayDuration)
+			uc := &testsuite.TestUpdateCallback{
+				OnReject: func(err error) {
+					require.Fail(t, "update should not be rejected")
+				},
+				OnAccept: func() {},
+				OnComplete: func(r any, err error) {
+					require.EqualValues(t, "Existing namespace delete delay timer is cancelled. Namespace delete delay is updated to 1h0m0s.", r)
+				},
+			}
+			env.UpdateWorkflow("update_namespace_delete_delay", "", uc, "1h")
+		}
+		if timerNo == 3 {
+			require.Equal(t, 1*time.Hour, delayDuration)
+			uc := &testsuite.TestUpdateCallback{
+				OnReject: func(err error) {
+					require.Fail(t, "update should not be rejected")
+				},
+				OnAccept: func() {},
+				OnComplete: func(r any, err error) {
+					require.EqualValues(t, "Existing namespace delete delay timer is cancelled. Namespace delete delay is removed.", r)
+				},
+			}
+			env.UpdateWorkflow("update_namespace_delete_delay", "", uc, "0")
+		}
+	})
+
+	// If the timer is not updated, WF will fail.
+	env.SetWorkflowRunTimeout(1 * time.Minute)
+
+	env.ExecuteWorkflow(ReclaimResourcesWorkflow, ReclaimResourcesParams{
+		DeleteExecutionsParams: deleteexecutions.DeleteExecutionsParams{
+			Namespace:            "namespace",
+			NamespaceID:          "namespace-id",
+			Config:               deleteexecutions.DeleteExecutionsConfig{},
+			PreviousSuccessCount: 0,
+			PreviousErrorCount:   0,
+		},
+		NamespaceDeleteDelay: 10 * time.Hour,
+	})
+
+	require.True(t, env.IsWorkflowCompleted())
+	require.NoError(t, env.GetWorkflowError())
+	var result ReclaimResourcesResult
+	require.NoError(t, env.GetWorkflowResult(&result))
+	require.Equal(t, 0, result.DeleteErrorCount)
+	require.Equal(t, 10, result.DeleteSuccessCount)
+	require.Equal(t, true, result.NamespaceDeleted)
+}