diff --git a/schema/elasticsearch/visibility/index_template_v7.json b/schema/elasticsearch/visibility/index_template_v7.json index 09e6026430..a1db5eb647 120000 --- a/schema/elasticsearch/visibility/index_template_v7.json +++ b/schema/elasticsearch/visibility/index_template_v7.json @@ -1 +1 @@ -./versioned/v13/index_template_v7.json \ No newline at end of file +./versioned/v14/index_template_v7.json \ No newline at end of file diff --git a/schema/elasticsearch/visibility/versioned/v14/index_template_v7.json b/schema/elasticsearch/visibility/versioned/v14/index_template_v7.json new file mode 100644 index 0000000000..3f442859c6 --- /dev/null +++ b/schema/elasticsearch/visibility/versioned/v14/index_template_v7.json @@ -0,0 +1,170 @@ +{ + "order": 0, + "index_patterns": ["temporal_visibility_v1*"], + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "0", + "auto_expand_replicas": "0-2", + "search.idle.after": "365d", + "sort.field": ["CloseTime", "StartTime", "RunId"], + "sort.order": ["desc", "desc", "desc"], + "sort.missing": ["_first", "_first", "_first"] + } + }, + "mappings": { + "dynamic": "false", + "properties": { + "NamespaceId": { + "type": "keyword" + }, + "TemporalNamespaceDivision": { + "type": "keyword" + }, + "WorkflowId": { + "type": "keyword" + }, + "RunId": { + "type": "keyword" + }, + "WorkflowType": { + "type": "keyword" + }, + "StartTime": { + "type": "date_nanos" + }, + "ExecutionTime": { + "type": "date_nanos" + }, + "CloseTime": { + "type": "date_nanos" + }, + "ExecutionDuration": { + "type": "long" + }, + "ExecutionStatus": { + "type": "keyword" + }, + "TaskQueue": { + "type": "keyword" + }, + "TemporalChangeVersion": { + "type": "keyword" + }, + "BatcherNamespace": { + "type": "keyword" + }, + "BatcherUser": { + "type": "keyword" + }, + "BinaryChecksums": { + "type": "keyword" + }, + "HistoryLength": { + "type": "long" + }, + "StateTransitionCount": { + "type": "long" + }, + "TemporalScheduledStartTime": { + "type": "date_nanos" + }, + "TemporalScheduledById": { + "type": "keyword" + }, + "TemporalSchedulePaused": { + "type": "boolean" + }, + "HistorySizeBytes": { + "type": "long" + }, + "BuildIds": { + "type": "keyword" + }, + "ParentWorkflowId": { + "type": "keyword" + }, + "ParentRunId": { + "type": "keyword" + }, + "RootWorkflowId": { + "type": "keyword" + }, + "RootRunId": { + "type": "keyword" + }, + "TemporalPauseInfo": { + "type": "keyword" + }, + "TemporalWorkerDeploymentVersion": { + "type": "keyword" + }, + "TemporalWorkflowVersioningBehavior": { + "type": "keyword" + }, + "TemporalWorkerDeployment": { + "type": "keyword" + }, + "TemporalReportedProblems": { + "type": "keyword" + }, + "TemporalUsedWorkerDeploymentVersions": { + "type": "keyword" + }, + "TemporalExternalPayloadSizeBytes": { + "type": "long" + }, + "TemporalExternalPayloadCount": { + "type": "long" + }, + "TemporalBool01": { + "type": "boolean" + }, + "TemporalBool02": { + "type": "boolean" + }, + "TemporalDatetime01": { + "type": "date_nanos" + }, + "TemporalDatetime02": { + "type": "date_nanos" + }, + "TemporalDouble01": { + "type": "scaled_float", + "scaling_factor": 10000 + }, + "TemporalDouble02": { + "type": "scaled_float", + "scaling_factor": 10000 + }, + "TemporalInt01": { + "type": "long" + }, + "TemporalInt02": { + "type": "long" + }, + "TemporalKeyword01": { + "type": "keyword" + }, + "TemporalKeyword02": { + "type": "keyword" + }, + "TemporalKeyword03": { + "type": "keyword" + }, + "TemporalKeyword04": { + "type": "keyword" + }, + "TemporalLowCardinalityKeyword01": { + "type": "keyword" + }, + "TemporalKeywordList01": { + "type": "keyword" + }, + "TemporalKeywordList02": { + "type": "keyword" + } + } + }, + "aliases": {} +} \ No newline at end of file diff --git a/schema/elasticsearch/visibility/versioned/v14/upgrade.sh b/schema/elasticsearch/visibility/versioned/v14/upgrade.sh new file mode 100755 index 0000000000..a968b73734 --- /dev/null +++ b/schema/elasticsearch/visibility/versioned/v14/upgrade.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +# Prerequisites: +# - jq +# - curl + +# Input parameters. +: "${ES_SCHEME:=http}" +: "${ES_SERVER:=127.0.0.1}" +: "${ES_PORT:=9200}" +: "${ES_USER:=}" +: "${ES_PWD:=}" +: "${ES_VERSION:=v7}" +: "${ES_VIS_INDEX_V1:=temporal_visibility_v1_dev}" +: "${AUTO_CONFIRM:=}" +: "${SLICES_COUNT:=auto}" + +es_endpoint="${ES_SCHEME}://${ES_SERVER}:${ES_PORT}" + +echo "=== Step 0. Sanity check if Elasticsearch index is accessible ===" + +if ! curl --silent --fail --user "${ES_USER}":"${ES_PWD}" "${es_endpoint}/${ES_VIS_INDEX_V1}/_stats/docs" --write-out "\n"; then + echo "Elasticsearch index ${ES_VIS_INDEX_V1} is not accessible at ${es_endpoint}." + exit 1 +fi + +echo "=== Step 1. Add TemporalExternalPayloadSizeBytes and TemporalExternalPayloadCount builtin search attributes ===" + +new_mapping=' +{ + "properties": { + "TemporalExternalPayloadSizeBytes": { + "type": "long" + }, + "TemporalExternalPayloadCount": { + "type": "long" + } + } +} +' + +if [ -z "${AUTO_CONFIRM}" ]; then + read -p "Add TemporalExternalPayloadSizeBytes and TemporalExternalPayloadCount builtin search attributes to the index ${ES_VIS_INDEX_V1}? (N/y)" -n 1 -r + echo +else + REPLY="y" +fi +if [ "${REPLY}" = "y" ]; then + curl --silent --fail --user "${ES_USER}":"${ES_PWD}" -X PUT "${es_endpoint}/${ES_VIS_INDEX_V1}/_mapping" -H "Content-Type: application/json" --data-binary "$new_mapping" | jq + # Wait for mapping changes to go through. + until curl --silent --user "${ES_USER}":"${ES_PWD}" "${es_endpoint}/_cluster/health/${ES_VIS_INDEX_V1}" | jq --exit-status '.status=="green" | .'; do + echo "Waiting for Elasticsearch index ${ES_VIS_INDEX_V1} become green." + sleep 1 + done +fi \ No newline at end of file diff --git a/schema/embed.go b/schema/embed.go index e994689eac..95f4758c56 100644 --- a/schema/embed.go +++ b/schema/embed.go @@ -58,7 +58,7 @@ func ElasticsearchClusterSettings() (string, error) { // ElasticsearchIndexTemplate returns the embedded index template for Elasticsearch v7 (latest version) func ElasticsearchIndexTemplate() (string, error) { - data, err := assets.ReadFile("elasticsearch/visibility/versioned/v13/index_template_v7.json") + data, err := assets.ReadFile("elasticsearch/visibility/versioned/v14/index_template_v7.json") if err != nil { return "", err } diff --git a/schema/mysql/v8/version.go b/schema/mysql/v8/version.go index ba5bd8c6a6..a7cfc51791 100644 --- a/schema/mysql/v8/version.go +++ b/schema/mysql/v8/version.go @@ -6,4 +6,4 @@ package v8 const Version = "1.19" // VisibilityVersion is the MySQL visibility database release version -const VisibilityVersion = "1.13" +const VisibilityVersion = "1.14" diff --git a/schema/mysql/v8/visibility/schema.sql b/schema/mysql/v8/visibility/schema.sql index 2d0ef647ff..68eedf5395 100644 --- a/schema/mysql/v8/visibility/schema.sql +++ b/schema/mysql/v8/visibility/schema.sql @@ -51,7 +51,8 @@ CREATE TABLE executions_visibility ( TemporalWorkflowVersioningBehavior VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalWorkflowVersioningBehavior"), TemporalWorkerDeployment VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalWorkerDeployment"), TemporalUsedWorkerDeploymentVersions JSON GENERATED ALWAYS AS (search_attributes->'$.TemporalUsedWorkerDeploymentVersions'), - + TemporalExternalPayloadSizeBytes BIGINT GENERATED ALWAYS AS (search_attributes->"$.TemporalExternalPayloadSizeBytes"), + TemporalExternalPayloadCount BIGINT GENERATED ALWAYS AS (search_attributes->"$.TemporalExternalPayloadCount"), PRIMARY KEY (namespace_id, run_id) ); @@ -85,7 +86,8 @@ CREATE INDEX by_temporal_scheduled_start_time ON executions_visibility (namespac CREATE INDEX by_temporal_scheduled_by_id ON executions_visibility (namespace_id, TemporalScheduledById, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); CREATE INDEX by_temporal_schedule_paused ON executions_visibility (namespace_id, TemporalSchedulePaused, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); CREATE INDEX by_temporal_namespace_division ON executions_visibility (namespace_id, TemporalNamespaceDivision, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); - +CREATE INDEX by_temporal_external_payload_size_bytes ON executions_visibility (namespace_id, TemporalExternalPayloadSizeBytes, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_external_payload_count ON executions_visibility (namespace_id, TemporalExternalPayloadCount, (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, start_time DESC, run_id); CREATE TABLE custom_search_attributes ( namespace_id CHAR(64) NOT NULL, diff --git a/schema/mysql/v8/visibility/versioned/v1.14/add_external_payload_size_and_count_search_attributes.sql b/schema/mysql/v8/visibility/versioned/v1.14/add_external_payload_size_and_count_search_attributes.sql new file mode 100644 index 0000000000..3b25819194 --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.14/add_external_payload_size_and_count_search_attributes.sql @@ -0,0 +1,25 @@ +ALTER TABLE executions_visibility +ADD COLUMN TemporalExternalPayloadSizeBytes BIGINT +GENERATED ALWAYS AS (search_attributes->"$.TemporalExternalPayloadSizeBytes"); + +ALTER TABLE executions_visibility +ADD COLUMN TemporalExternalPayloadCount BIGINT +GENERATED ALWAYS AS (search_attributes->"$.TemporalExternalPayloadCount"); + +CREATE INDEX by_temporal_external_payload_size_bytes +ON executions_visibility ( + namespace_id, + TemporalExternalPayloadSizeBytes, + (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, + start_time DESC, + run_id +); + +CREATE INDEX by_temporal_external_payload_count +ON executions_visibility ( + namespace_id, + TemporalExternalPayloadCount, + (COALESCE(close_time, CAST('9999-12-31 23:59:59' AS DATETIME))) DESC, + start_time DESC, + run_id +); \ No newline at end of file diff --git a/schema/mysql/v8/visibility/versioned/v1.14/manifest.json b/schema/mysql/v8/visibility/versioned/v1.14/manifest.json new file mode 100644 index 0000000000..99ed79e480 --- /dev/null +++ b/schema/mysql/v8/visibility/versioned/v1.14/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.14", + "MinCompatibleVersion": "0.1", + "Description": "add TemporalExternalPayloadSizeBytes and TemporalExternalPayloadCount builtin search attributes", + "SchemaUpdateCqlFiles": [ + "add_external_payload_size_and_count_search_attributes.sql" + ] +} \ No newline at end of file diff --git a/schema/postgresql/v12/version.go b/schema/postgresql/v12/version.go index 3aaff1c714..77e0f761e0 100644 --- a/schema/postgresql/v12/version.go +++ b/schema/postgresql/v12/version.go @@ -8,4 +8,4 @@ const Version = "1.19" // VisibilityVersion is the Postgres visibility database release version // Temporal supports both MySQL and Postgres officially, so upgrade should be performed for both MySQL and Postgres -const VisibilityVersion = "1.13" +const VisibilityVersion = "1.14" diff --git a/schema/postgresql/v12/visibility/schema.sql b/schema/postgresql/v12/visibility/schema.sql index e681dcd24d..501612b857 100644 --- a/schema/postgresql/v12/visibility/schema.sql +++ b/schema/postgresql/v12/visibility/schema.sql @@ -56,6 +56,8 @@ CREATE TABLE executions_visibility ( TemporalWorkflowVersioningBehavior VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>'TemporalWorkflowVersioningBehavior') STORED, TemporalWorkerDeployment VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>'TemporalWorkerDeployment') STORED, TemporalUsedWorkerDeploymentVersions JSONB GENERATED ALWAYS AS (search_attributes->'TemporalUsedWorkerDeploymentVersions') STORED, + TemporalExternalPayloadSizeBytes BIGINT GENERATED ALWAYS AS ((search_attributes->'TemporalExternalPayloadSizeBytes')::bigint) STORED, + TemporalExternalPayloadCount BIGINT GENERATED ALWAYS AS ((search_attributes->'TemporalExternalPayloadCount')::bigint) STORED, -- Pre-allocated custom search attributes Bool01 BOOLEAN GENERATED ALWAYS AS ((search_attributes->'Bool01')::boolean) STORED, @@ -137,6 +139,8 @@ CREATE INDEX by_temporal_scheduled_start_time ON executions_visibility (namespac CREATE INDEX by_temporal_scheduled_by_id ON executions_visibility (namespace_id, TemporalScheduledById, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); CREATE INDEX by_temporal_schedule_paused ON executions_visibility (namespace_id, TemporalSchedulePaused, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); CREATE INDEX by_temporal_namespace_division ON executions_visibility (namespace_id, TemporalNamespaceDivision, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_external_payload_size_bytes ON executions_visibility (namespace_id, TemporalExternalPayloadSizeBytes, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); +CREATE INDEX by_temporal_external_payload_count ON executions_visibility (namespace_id, TemporalExternalPayloadCount, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); -- Indexes for the pre-allocated custom search attributes CREATE INDEX by_bool_01 ON executions_visibility (namespace_id, Bool01, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); diff --git a/schema/postgresql/v12/visibility/versioned/v1.14/add_external_payload_size_and_count_search_attributes.sql b/schema/postgresql/v12/visibility/versioned/v1.14/add_external_payload_size_and_count_search_attributes.sql new file mode 100644 index 0000000000..8e46782d7c --- /dev/null +++ b/schema/postgresql/v12/visibility/versioned/v1.14/add_external_payload_size_and_count_search_attributes.sql @@ -0,0 +1,9 @@ +ALTER TABLE executions_visibility +ADD COLUMN TemporalExternalPayloadSizeBytes BIGINT GENERATED ALWAYS AS ((search_attributes->'TemporalExternalPayloadSizeBytes')::bigint) STORED; + +ALTER TABLE executions_visibility +ADD COLUMN TemporalExternalPayloadCount BIGINT GENERATED ALWAYS AS ((search_attributes->'TemporalExternalPayloadCount')::bigint) STORED; + +CREATE INDEX by_temporal_external_payload_size_bytes ON executions_visibility (namespace_id, TemporalExternalPayloadSizeBytes, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); + +CREATE INDEX by_temporal_external_payload_count ON executions_visibility (namespace_id, TemporalExternalPayloadCount, (COALESCE(close_time, '9999-12-31 23:59:59')) DESC, start_time DESC, run_id); diff --git a/schema/postgresql/v12/visibility/versioned/v1.14/manifest.json b/schema/postgresql/v12/visibility/versioned/v1.14/manifest.json new file mode 100644 index 0000000000..99ed79e480 --- /dev/null +++ b/schema/postgresql/v12/visibility/versioned/v1.14/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "1.14", + "MinCompatibleVersion": "0.1", + "Description": "add TemporalExternalPayloadSizeBytes and TemporalExternalPayloadCount builtin search attributes", + "SchemaUpdateCqlFiles": [ + "add_external_payload_size_and_count_search_attributes.sql" + ] +} \ No newline at end of file diff --git a/schema/sqlite/v3/visibility/schema.sql b/schema/sqlite/v3/visibility/schema.sql index 8043711cc1..74e231f43d 100644 --- a/schema/sqlite/v3/visibility/schema.sql +++ b/schema/sqlite/v3/visibility/schema.sql @@ -36,6 +36,8 @@ CREATE TABLE executions_visibility ( TemporalWorkflowVersioningBehavior VARCHAR(255) GENERATED ALWAYS AS (JSON_EXTRACT(search_attributes, "$.TemporalWorkflowVersioningBehavior")), TemporalWorkerDeployment VARCHAR(255) GENERATED ALWAYS AS (JSON_EXTRACT(search_attributes, "$.TemporalWorkerDeployment")), TemporalUsedWorkerDeploymentVersions TEXT GENERATED ALWAYS AS (JSON_EXTRACT(search_attributes, "$.TemporalUsedWorkerDeploymentVersions")) STORED, + TemporalExternalPayloadSizeBytes BIGINT GENERATED ALWAYS AS (JSON_EXTRACT(search_attributes, "$.TemporalExternalPayloadSizeBytes")) STORED, + TemporalExternalPayloadCount BIGINT GENERATED ALWAYS AS (JSON_EXTRACT(search_attributes, "$.TemporalExternalPayloadCount")) STORED, -- Pre-allocated custom search attributes Bool01 BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(search_attributes, "$.Bool01")),